From dfe44abeff5a9960bfa542077834a0a863a8fdaf Mon Sep 17 00:00:00 2001 From: Douglas McConnachie Date: Fri, 13 Mar 2020 12:46:21 +0000 Subject: [PATCH] feat(rest): add openapi schema consolidation Add openapi schema enhancer to rest server. Consolidates openapi schema, by creating references to schema to reduce duplication. Can be disabled by setting rest option openApiSpec.consolidate to false. Signed-off-by: Douglas McConnachie --- packages/openapi-v3/package-lock.json | 29 ++ packages/openapi-v3/package.json | 1 + packages/rest/package-lock.json | 23 ++ packages/rest/package.json | 2 + .../consolidate.spec.extension.unit.ts | 281 ++++++++++++++++++ .../rest.server.open-api-spec.unit.ts | 60 +++- packages/rest/src/rest.component.ts | 2 + packages/rest/src/rest.server.ts | 6 + .../consolidate.spec-enhancer.ts | 182 ++++++++++++ 9 files changed, 584 insertions(+), 2 deletions(-) create mode 100644 packages/rest/src/__tests__/unit/rest.server/consolidate.spec.extension.unit.ts create mode 100644 packages/rest/src/spec-enhancers/consolidate.spec-enhancer.ts diff --git a/packages/openapi-v3/package-lock.json b/packages/openapi-v3/package-lock.json index 986c5a997cfb..b8bca8807789 100644 --- a/packages/openapi-v3/package-lock.json +++ b/packages/openapi-v3/package-lock.json @@ -19,6 +19,27 @@ "http-status": "*" } }, + "@types/json-merge-patch": { + "version": "0.0.4", + "resolved": "https://registry.npmjs.org/@types/json-merge-patch/-/json-merge-patch-0.0.4.tgz", + "integrity": "sha1-pSgtqWkKgSpiEoo0cIr0dqMI2UE=", + "dev": true + }, + "@types/json-schema": { + "version": "7.0.4", + "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.4.tgz", + "integrity": "sha512-8+KAKzEvSUdeo+kmqnKrqgeE+LcA0tjYWFY7RPProVYwnqDjukzO+3b6dLD56rYX5TdWejnEOLJYOIeh4CXKuA==", + "dev": true + }, + "@types/json-schema-compare": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/@types/json-schema-compare/-/json-schema-compare-0.2.0.tgz", + "integrity": "sha512-TtCXQjsCQi+fcandEbzDJhqyztpVM9c5mtGuk7Hf8yQsdaBpfjEkOicfydAEWB684wGCzUrV5ttvt9hCyDCoxA==", + "dev": true, + "requires": { + "@types/json-schema": "*" + } + }, "@types/lodash": { "version": "4.14.149", "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.149.tgz", @@ -150,6 +171,14 @@ "deep-equal": "^1.0.0" } }, + "json-schema-compare": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/json-schema-compare/-/json-schema-compare-0.2.2.tgz", + "integrity": "sha512-c4WYmDKyJXhs7WWvAWm3uIYnfyWFoIp+JEoX34rctVvEkMYCPGhXtvmFFXiffBbxfZsvQ0RNnV5H7GvDF5HCqQ==", + "requires": { + "lodash": "^4.17.4" + } + }, "lodash": { "version": "4.17.15", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz", diff --git a/packages/openapi-v3/package.json b/packages/openapi-v3/package.json index ae863c7209f3..df78efd7a41a 100644 --- a/packages/openapi-v3/package.json +++ b/packages/openapi-v3/package.json @@ -23,6 +23,7 @@ "@loopback/testlab": "^3.0.1", "@types/debug": "^4.1.5", "@types/http-status": "^1.1.2", + "@types/json-merge-patch": "0.0.4", "@types/lodash": "^4.14.149", "@types/node": "^10.17.19" }, diff --git a/packages/rest/package-lock.json b/packages/rest/package-lock.json index 03eb456297d1..dc3d7c055877 100644 --- a/packages/rest/package-lock.json +++ b/packages/rest/package-lock.json @@ -95,6 +95,21 @@ "integrity": "sha512-otRe77JNNWzoVGLKw8TCspKswRoQToys4tuL6XYVBFxjgeM0RUrx7m3jkaTdxILxeGry3zM8mGYkGXMeQ02guA==", "dev": true }, + "@types/json-schema": { + "version": "7.0.4", + "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.4.tgz", + "integrity": "sha512-8+KAKzEvSUdeo+kmqnKrqgeE+LcA0tjYWFY7RPProVYwnqDjukzO+3b6dLD56rYX5TdWejnEOLJYOIeh4CXKuA==", + "dev": true + }, + "@types/json-schema-compare": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/@types/json-schema-compare/-/json-schema-compare-0.2.0.tgz", + "integrity": "sha512-TtCXQjsCQi+fcandEbzDJhqyztpVM9c5mtGuk7Hf8yQsdaBpfjEkOicfydAEWB684wGCzUrV5ttvt9hCyDCoxA==", + "dev": true, + "requires": { + "@types/json-schema": "*" + } + }, "@types/lodash": { "version": "4.14.149", "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.149.tgz", @@ -845,6 +860,14 @@ "xmlcreate": "^2.0.3" } }, + "json-schema-compare": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/json-schema-compare/-/json-schema-compare-0.2.2.tgz", + "integrity": "sha512-c4WYmDKyJXhs7WWvAWm3uIYnfyWFoIp+JEoX34rctVvEkMYCPGhXtvmFFXiffBbxfZsvQ0RNnV5H7GvDF5HCqQ==", + "requires": { + "lodash": "^4.17.4" + } + }, "json-schema-traverse": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", diff --git a/packages/rest/package.json b/packages/rest/package.json index 635ebbd79950..aef87285da9f 100644 --- a/packages/rest/package.json +++ b/packages/rest/package.json @@ -40,6 +40,7 @@ "debug": "^4.1.1", "express": "^4.17.1", "http-errors": "^1.7.3", + "json-schema-compare": "^0.2.2", "js-yaml": "^3.13.1", "lodash": "^4.17.15", "on-finished": "^2.3.0", @@ -57,6 +58,7 @@ "@loopback/repository": "^2.1.1", "@loopback/testlab": "^3.0.1", "@types/debug": "^4.1.5", + "@types/json-schema-compare": "^0.2.0", "@types/js-yaml": "^3.12.3", "@types/lodash": "^4.14.149", "@types/multer": "^1.4.2", diff --git a/packages/rest/src/__tests__/unit/rest.server/consolidate.spec.extension.unit.ts b/packages/rest/src/__tests__/unit/rest.server/consolidate.spec.extension.unit.ts new file mode 100644 index 000000000000..b78cab62cb14 --- /dev/null +++ b/packages/rest/src/__tests__/unit/rest.server/consolidate.spec.extension.unit.ts @@ -0,0 +1,281 @@ +// Copyright IBM Corp. 2020. All Rights Reserved. +// Node module: @loopback/openapi-v3 +// This file is licensed under the MIT License. +// License text available at https://opensource.org/licenses/MIT + +import { + ComponentsSpecBuilder, + OpenApiSpecBuilder, + OperationSpecBuilder, +} from '@loopback/openapi-spec-builder'; +import {expect} from '@loopback/testlab'; +import {ConsolidationEnhancer} from '../../../spec-enhancers/consolidate.spec-enhancer'; + +const consolidationEnhancer = new ConsolidationEnhancer(); + +describe('consolidateSchemaObjects', () => { + it('moves schema with title to component.schemas, replaces with reference', () => { + const INPUT_SPEC = new OpenApiSpecBuilder() + .withOperation( + 'get', + '/', + new OperationSpecBuilder().withResponse(200, { + description: 'Example', + content: { + 'application/json': { + schema: { + title: 'loopback.example', + properties: { + test: { + type: 'string', + }, + }, + }, + }, + }, + }), + ) + .build(); + + const EXPECTED_SPEC = new OpenApiSpecBuilder() + .withOperation( + 'get', + '/', + new OperationSpecBuilder().withResponse(200, { + description: 'Example', + content: { + 'application/json': { + schema: { + $ref: '#/components/schemas/loopback.example', + }, + }, + }, + }), + ) + .withComponents( + new ComponentsSpecBuilder().withSchema('loopback.example', { + title: 'loopback.example', + properties: { + test: { + type: 'string', + }, + }, + }), + ) + .build(); + + expect(consolidationEnhancer.modifySpec(INPUT_SPEC)).to.eql(EXPECTED_SPEC); + }); + + it('ignores schema without title property', () => { + const INPUT_SPEC = new OpenApiSpecBuilder() + .withOperation( + 'get', + '/', + new OperationSpecBuilder().withResponse(200, { + description: 'Example', + content: { + 'application/json': { + schema: { + properties: { + test: { + type: 'string', + }, + }, + }, + }, + }, + }), + ) + .build(); + + expect(consolidationEnhancer.modifySpec(INPUT_SPEC)).to.eql(INPUT_SPEC); + }); + + it('avoids naming collision', () => { + const INPUT_SPEC = new OpenApiSpecBuilder() + .withOperation( + 'get', + '/', + new OperationSpecBuilder().withResponse(200, { + description: 'Example', + content: { + 'application/json': { + schema: { + title: 'loopback.example', + properties: { + test: { + type: 'string', + }, + }, + }, + }, + }, + }), + ) + .withComponents( + new ComponentsSpecBuilder().withSchema('loopback.example', { + title: 'Different loopback.example exists', + properties: { + testDiff: { + type: 'string', + }, + }, + }), + ) + .build(); + + const EXPECTED_SPEC = new OpenApiSpecBuilder() + .withOperation( + 'get', + '/', + new OperationSpecBuilder().withResponse(200, { + description: 'Example', + content: { + 'application/json': { + schema: { + $ref: '#/components/schemas/loopback.example1', + }, + }, + }, + }), + ) + .withComponents( + new ComponentsSpecBuilder() + .withSchema('loopback.example', { + title: 'Different loopback.example exists', + properties: { + testDiff: { + type: 'string', + }, + }, + }) + .withSchema('loopback.example1', { + title: 'loopback.example', + properties: { + test: { + type: 'string', + }, + }, + }), + ) + .build(); + + expect(consolidationEnhancer.modifySpec(INPUT_SPEC)).to.eql(EXPECTED_SPEC); + }); + + it('consolidates same schema in multiple locations', () => { + const INPUT_SPEC = new OpenApiSpecBuilder() + .withOperation( + 'get', + // first time has 'loopback.example' + '/path1', + new OperationSpecBuilder().withResponse(200, { + description: 'Example', + content: { + 'application/json': { + schema: { + title: 'loopback.example', + properties: { + test: { + type: 'string', + }, + }, + }, + }, + }, + }), + ) + .withOperation( + 'get', + // second time has 'loopback.example' + '/path2', + new OperationSpecBuilder().withResponse(200, { + description: 'Example', + content: { + 'application/json': { + schema: { + title: 'loopback.example', + properties: { + test: { + type: 'string', + }, + }, + }, + }, + }, + }), + ) + .build(); + + const EXPECTED_SPEC = new OpenApiSpecBuilder() + .withOperation( + 'get', + '/path1', + new OperationSpecBuilder().withResponse(200, { + description: 'Example', + content: { + 'application/json': { + schema: { + $ref: '#/components/schemas/loopback.example', + }, + }, + }, + }), + ) + .withOperation( + 'get', + '/path2', + new OperationSpecBuilder().withResponse(200, { + description: 'Example', + content: { + 'application/json': { + schema: { + $ref: '#/components/schemas/loopback.example', + }, + }, + }, + }), + ) + .withComponents( + new ComponentsSpecBuilder().withSchema('loopback.example', { + title: 'loopback.example', + properties: { + test: { + type: 'string', + }, + }, + }), + ) + .build(); + + expect(consolidationEnhancer.modifySpec(INPUT_SPEC)).to.eql(EXPECTED_SPEC); + }); + + it('obeys disabled option when set to true', () => { + consolidationEnhancer.disabled = true; + const INPUT_SPEC = new OpenApiSpecBuilder() + .withOperation( + 'get', + '/', + new OperationSpecBuilder().withResponse(200, { + description: 'Example', + content: { + 'application/json': { + schema: { + title: 'loopback.example', + properties: { + test: { + type: 'string', + }, + }, + }, + }, + }, + }), + ) + .build(); + + expect(consolidationEnhancer.modifySpec(INPUT_SPEC)).to.eql(INPUT_SPEC); + }); +}); diff --git a/packages/rest/src/__tests__/unit/rest.server/rest.server.open-api-spec.unit.ts b/packages/rest/src/__tests__/unit/rest.server/rest.server.open-api-spec.unit.ts index ad7930ef4798..8e9b22c1f4ec 100644 --- a/packages/rest/src/__tests__/unit/rest.server/rest.server.open-api-spec.unit.ts +++ b/packages/rest/src/__tests__/unit/rest.server/rest.server.open-api-spec.unit.ts @@ -4,16 +4,27 @@ // License text available at https://opensource.org/licenses/MIT import {Application, createBindingFromClass} from '@loopback/core'; -import {anOpenApiSpec, anOperationSpec} from '@loopback/openapi-spec-builder'; +import { + anOpenApiSpec, + anOperationSpec, + OpenApiSpecBuilder, + OperationSpecBuilder, +} from '@loopback/openapi-spec-builder'; import {get, post, requestBody} from '@loopback/openapi-v3'; import {model, property} from '@loopback/repository'; -import {expect, validateApiSpec} from '@loopback/testlab'; +import { + expect, + givenHttpServerConfig, + validateApiSpec, +} from '@loopback/testlab'; import { createControllerFactoryForClass, RestComponent, RestServer, + RestServerConfig, } from '../../..'; import {RestTags} from '../../../keys'; +import {ConsolidationEnhancer} from '../../../spec-enhancers/consolidate.spec-enhancer'; import {TestInfoSpecEnhancer} from './fixtures/info.spec.extension'; describe('RestServer.getApiSpec()', () => { @@ -321,6 +332,11 @@ describe('RestServer.getApiSpec()', () => { }); }); + it('registers consolidate enhancer', async () => { + const enhancer = await server.OASEnhancer.getEnhancerByName('consolidate'); + expect(enhancer).to.be.instanceOf(ConsolidationEnhancer); + }); + it('invokes registered oas enhancers', async () => { const EXPECTED_SPEC_INFO = { title: 'LoopBack Test Application', @@ -393,6 +409,46 @@ describe('RestServer.getApiSpec()', () => { expect(spec.info).to.eql(EXPECTED_SPEC_INFO); }); + context('options', () => { + it('disables consolidator if consolidate is set to false', async () => { + const options: {rest: RestServerConfig} = { + rest: {openApiSpec: {consolidate: false}}, + }; + options.rest = givenHttpServerConfig(options.rest); + app = new Application(options); + app.component(RestComponent); + server = await app.getServer(RestServer); + await server.start(); + + const INPUT_SPEC = new OpenApiSpecBuilder() + .withOperation( + 'get', + '/', + new OperationSpecBuilder().withResponse(200, { + description: 'Example', + content: { + 'application/json': { + schema: { + title: 'loopback.example', + properties: { + test: { + type: 'string', + }, + }, + }, + }, + }, + }), + ) + .build(); + server.api(INPUT_SPEC); + const spec = await server.getApiSpec(); + expect(spec).to.eql(INPUT_SPEC); + + await server.stop(); + }); + }); + async function givenApplication() { app = new Application(); app.component(RestComponent); diff --git a/packages/rest/src/rest.component.ts b/packages/rest/src/rest.component.ts index d2a1b3f2142e..53e4fde8251c 100644 --- a/packages/rest/src/rest.component.ts +++ b/packages/rest/src/rest.component.ts @@ -42,6 +42,7 @@ import { RestServerConfig, } from './rest.server'; import {DefaultSequence} from './sequence'; +import {ConsolidationEnhancer} from './spec-enhancers/consolidate.spec-enhancer'; import {InfoSpecEnhancer} from './spec-enhancers/info.spec-enhancer'; import {AjvFactoryProvider} from './validation/ajv-factory.provider'; @@ -85,6 +86,7 @@ export class RestComponent implements Component { RestBindings.REQUEST_BODY_PARSER_STREAM, ), createBindingFromClass(InfoSpecEnhancer), + createBindingFromClass(ConsolidationEnhancer), ]; servers: { [name: string]: Constructor; diff --git a/packages/rest/src/rest.server.ts b/packages/rest/src/rest.server.ts index 33fd29fe81dc..082a4f25bbb6 100644 --- a/packages/rest/src/rest.server.ts +++ b/packages/rest/src/rest.server.ts @@ -1066,6 +1066,12 @@ export interface OpenApiSpecOptions { * Set this flag to disable the endpoint for OpenAPI spec */ disabled?: true; + + /** + * Set this flag to `false` to disable OAS schema consolidation. If not set, + * the value defaults to `true`. + */ + consolidate?: boolean; } export interface ApiExplorerOptions { diff --git a/packages/rest/src/spec-enhancers/consolidate.spec-enhancer.ts b/packages/rest/src/spec-enhancers/consolidate.spec-enhancer.ts new file mode 100644 index 000000000000..cf4ff32a8ced --- /dev/null +++ b/packages/rest/src/spec-enhancers/consolidate.spec-enhancer.ts @@ -0,0 +1,182 @@ +// Copyright IBM Corp. 2020. All Rights Reserved. +// Node module: @loopback/rest +// This file is licensed under the MIT License. +// License text available at https://opensource.org/licenses/MIT + +import { + ApplicationConfig, + bind, + BindingScope, + CoreBindings, + inject, +} from '@loopback/core'; +import { + asSpecEnhancer, + ISpecificationExtension, + isSchemaObject, + OASEnhancer, + OpenApiSpec, + ReferenceObject, + SchemaObject, +} from '@loopback/openapi-v3'; +import debugFactory from 'debug'; +import compare from 'json-schema-compare'; +import _ from 'lodash'; + +const debug = debugFactory('loopback:openapi:spec-enhancer:consolidate'); + +/** + * This enhancer consolidates schemas into `/components/schemas` and replaces + * instances of said schema with a $ref pointer. + * + * Please note that the title property must be set on a schema in order to be + * considered for consolidation. + * + * For example, with the following schema instance: + * + * ```json + * schema: { + * title: 'loopback.example', + * properties: { + * test: { + * type: 'string', + * }, + * }, + * } + * ``` + * + * The consolidator will copy the schema body to + * `/components/schemas/loopback.example` and replace any instance of the schema + * with a reference to the component schema as follows: + * + * ```json + * schema: { + * $ref: '#/components/schemas/loopback.example', + * } + * ``` + * + * When comparing schemas to avoid naming collisions, the description field + * is ignored. + */ +@bind(asSpecEnhancer, {scope: BindingScope.SINGLETON}) +export class ConsolidationEnhancer implements OASEnhancer { + name = 'consolidate'; + disabled: boolean; + + constructor( + @inject(CoreBindings.APPLICATION_CONFIG, {optional: true}) + readonly config?: ApplicationConfig, + ) { + this.disabled = !(this.config?.rest?.openApiSpec?.consolidate || true); + } + + modifySpec(spec: OpenApiSpec): OpenApiSpec { + return !this.disabled ? this.consolidateSchemaObjects(spec) : spec; + } + + /** + * Recursively search OpenApiSpec PathsObject for SchemaObjects with title + * property. Moves reusable schema bodies to #/components/schemas and replace + * with json pointer. It handles title collisions with schema body comparision. + */ + private consolidateSchemaObjects(spec: OpenApiSpec): OpenApiSpec { + // use 'paths' as crawl root + this.recursiveWalk(spec.paths, ['paths'], spec); + + return spec; + } + + private recursiveWalk( + rootSchema: ISpecificationExtension, + parentPath: Array, + spec: OpenApiSpec, + ) { + if (this.isTraversable(rootSchema)) { + Object.entries(rootSchema).forEach(([key, subSchema]) => { + if (subSchema) { + this.recursiveWalk(subSchema, parentPath.concat(key), spec); + this.processSchema(subSchema, parentPath.concat(key), spec); + } + }); + } + } + + /** + * Carry out schema consolidation after tree traversal. If 'title' property + * set then we consider current schema for consolidation. SchemaObjects with + * properties (and title set) are moved to #/components/schemas/ and + * replaced with ReferenceObject. + * + * Features: + * - name collision protection + * + * @param schema - current schema element to process + * @param parentPath - path object to parent + * @param spec - subject OpenApi specification + */ + private processSchema( + schema: SchemaObject | ReferenceObject, + parentPath: Array<string>, + spec: OpenApiSpec, + ) { + const schemaObj = this.ifConsolidationCandidate(schema); + if (schemaObj) { + // name collison protection + let instanceNo = 1; + let title = schemaObj.title!; + let refSchema = this.getRefSchema(title, spec); + while ( + refSchema && + !compare(schemaObj as ISpecificationExtension, refSchema, { + ignore: ['description'], + }) + ) { + title = `${schemaObj.title}${instanceNo++}`; + refSchema = this.getRefSchema(title, spec); + } + if (!refSchema) { + debug('Creating new component $ref with schema %j', schema); + this.patchRef(title, schema, spec); + } + debug('Creating link to $ref %j', title); + this.patchPath(title, parentPath, spec); + } + } + + private getRefSchema( + name: string, + spec: OpenApiSpec, + ): ISpecificationExtension | undefined { + const schema = _.get(spec, ['components', 'schemas', name]); + + return schema; + } + + private patchRef( + name: string, + value: ISpecificationExtension, + spec: OpenApiSpec, + ) { + _.set(spec, ['components', 'schemas', name], value); + } + + private patchPath(name: string, path: Array<string>, spec: OpenApiSpec) { + const patch = { + $ref: `#/components/schemas/${name}`, + }; + _.set(spec, path, patch); + } + + private ifConsolidationCandidate( + schema: SchemaObject | ReferenceObject, + ): SchemaObject | undefined { + // use title to discriminate references + return isSchemaObject(schema) && schema.properties && schema.title + ? schema + : undefined; + } + + private isTraversable(schema: ISpecificationExtension): boolean { + return schema && typeof schema === 'object' ? true : false; + } +}