refactor(test): rewrite test mode validation

re #51

acceptance-tests/config-helpers.ts

@@ -78,7 +78,7 @@ export class ConfigBuilder {
     return this
   }
 
-  public withType(typeName: string): ConfigBuilder {
+  public withResponseType(typeName: string): ConfigBuilder {
     if (this.config.response.type) {
       throw new Error('Response type already set to ' + this.config.response.type)
     }

acceptance-tests/serve.spec.ts

@@ -315,7 +315,7 @@ describe('ncdc serve', () => {
 
       it('serves when the type matches the body', async () => {
         configWrapper = new ServeConfigWrapper()
-          .addConfig(new ConfigBuilder().withType(schemaName).withBody('Hello!').build())
+          .addConfig(new ConfigBuilder().withResponseType(schemaName).withBody('Hello!').build())
           .addSchemaFile(schemaName, { type: 'string' })
 
         serve = await prepareServe(typecheckingCleanup)(
@@ -354,7 +354,7 @@ describe('ncdc serve', () => {
 
       it('it serves when the type matches the body', async () => {
         configWrapper = new ServeConfigWrapper()
-          .addConfig(new ConfigBuilder().withType('Book').build())
+          .addConfig(new ConfigBuilder().withResponseType('Book').build())
           .addType('Book', {
             ISBN: 'string',
             ISBN_13: 'string',

acceptance-tests/test.spec.ts

@@ -4,7 +4,7 @@ import { Server } from 'http'
 import strip from 'strip-ansi'
 
 jest.useRealTimers()
-jest.setTimeout(45000)
+jest.setTimeout(7000)
 
 describe('ncdc test', () => {
   let realServer: Server
@@ -33,15 +33,18 @@ describe('ncdc test', () => {
 
   it('can test endpoints that return json', async () => {
     realServer = new RealServerBuilder().withGetEndpoint('/api/resource', 200, { hello: 'world' }).start()
-    new TestConfigWrapper().addConfig(
-      new ConfigBuilder()
-        .withName('Hello')
-        .withEndpoints('/api/resource')
-        .withServeBody(undefined)
-        .withBody({ hello: 'world' })
-        .withResponseHeaders({ 'content-type': 'application/json' })
-        .build(),
-    )
+    new TestConfigWrapper()
+      .addConfig(
+        new ConfigBuilder()
+          .withName('Hello')
+          .withEndpoints('/api/resource')
+          .withServeBody(undefined)
+          .withBody({ hello: 'world' })
+          .withResponseType('Hello')
+          .withResponseHeaders({ 'content-type': 'application/json' })
+          .build(),
+      )
+      .addType('Hello', { hello: 'string' })
 
     const output = await runTestCommand()
 

src/commands/test/command.ts

@@ -3,7 +3,7 @@ import { Argv, CommandModule } from 'yargs'
 import * as consts from '~commands/consts'
 import { createHandler, TestArgs } from './handler'
 import logger from '~logger'
-import { testConfigs } from './test'
+import { runTests } from './test'
 import loadConfig from '~config/load'
 
 const builder = (yargs: Argv): Argv<TestArgs> =>
@@ -46,6 +46,6 @@ export default function createTestCommand(
     command: 'test <configPath> <baseURL>',
     describe: 'Tests configured endpoints',
     builder,
-    handler: createHandler(handleError, createTypeValidator, logger, testConfigs, loadConfig),
+    handler: createHandler(handleError, createTypeValidator, logger, runTests, loadConfig),
   }
 }

src/commands/test/handler.spec.ts

@@ -1,11 +1,11 @@
-import { mockFn, randomString, mocked, mockObj } from '~test-helpers'
+import { mockFn, randomString, mocked, mockObj, randomNumber } from '~test-helpers'
 import { HandleError, CreateTypeValidator } from '~commands/shared'
 import { createHandler, TestArgs } from './handler'
 import { resolve } from 'path'
 import { existsSync } from 'fs'
 import { TypeValidator } from '~validation'
 import { NCDCLogger } from '~logger'
-import { TestConfigs } from './test'
+import { RunTests } from './test'
 import { LoadConfig, LoadConfigStatus } from '~config/load'
 import { ValidatedTestConfig, transformConfigs } from './config'
 import { ConfigBuilder } from '~config/types'
@@ -21,13 +21,13 @@ const mockedExistsSync = mocked(existsSync)
 const mockedResolve = mocked(resolve)
 const resolvedTsconfigPath = randomString('resolved-tsconfig')
 const mockedLogger = mockObj<NCDCLogger>({ warn: jest.fn() })
-const mockedTestConfigs = mockFn<TestConfigs>()
+const mockedRunTests = mockFn<RunTests>()
 const mockedLoadConfig = mockFn<LoadConfig<ValidatedTestConfig>>()
 const handler = createHandler(
   mockedHandleError,
   mockedCreateTypeValidator,
   mockedLogger,
-  mockedTestConfigs,
+  mockedRunTests,
   mockedLoadConfig,
 )
 
@@ -76,6 +76,18 @@ it('calls loadConfig with the correct args', async () => {
   expect(mockedLoadConfig).toBeCalledWith(args.configPath, expect.any(Function), transformConfigs)
 })
 
+it('only creates a type validator once for loading configs', async () => {
+  mockedLoadConfig.mockResolvedValue({ type: LoadConfigStatus.NoConfigs })
+  await handler(args)
+
+  const getTypeValidatorFn = mockedLoadConfig.mock.calls[0][1]
+  const timesToCall = randomNumber(1, 10)
+  for (let i = 0; i < timesToCall; i++) {
+    getTypeValidatorFn()
+  }
+  expect(mockedCreateTypeValidator).toBeCalledTimes(1)
+})
+
 const badStatuses = [
   LoadConfigStatus.InvalidBodies,
   LoadConfigStatus.InvalidConfig,
@@ -102,38 +114,45 @@ it('handles there being no configs to run as an error', async () => {
   )
 })
 
-it('calls testConfigs with the correct arguments', async () => {
+it('calls runTests with the correct arguments', async () => {
   const configs = [new ConfigBuilder().build()]
-  mockedLoadConfig.mockImplementation((_, getTypeValidator) => {
-    getTypeValidator()
-    return Promise.resolve({ type: LoadConfigStatus.Success, configs, absoluteFixturePaths: [] })
-  })
+  mockedLoadConfig.mockResolvedValue({ type: LoadConfigStatus.Success, configs, absoluteFixturePaths: [] })
 
   await handler(args)
 
   expect(mockedHandleError).not.toBeCalled()
-  expect(mockedTestConfigs).toBeCalledWith(
+  expect(mockedRunTests).toBeCalledWith(
     args.baseURL,
     expect.objectContaining({}),
     configs,
-    mockedTypeValidator,
+    expect.any(Function),
+    mockedLogger,
   )
 })
 
+it('only creates a type validator once for running the tests', async () => {
+  const configs = [new ConfigBuilder().build()]
+  mockedLoadConfig.mockResolvedValue({ type: LoadConfigStatus.Success, configs, absoluteFixturePaths: [] })
+
+  await handler(args)
+
+  const getTypeValidatorFn = mockedRunTests.mock.calls[0][3]
+  const timesToCall = randomNumber(1, 10)
+  for (let i = 0; i < timesToCall; i++) {
+    getTypeValidatorFn()
+  }
+  expect(mockedCreateTypeValidator).toBeCalledTimes(1)
+})
+
 it('handles errors thrown by testConfigs', async () => {
   mockedLoadConfig.mockResolvedValue({
     type: LoadConfigStatus.Success,
     absoluteFixturePaths: [],
     configs: [new ConfigBuilder().build()],
   })
-  mockedTestConfigs.mockRejectedValue(new Error('oops'))
+  mockedRunTests.mockResolvedValue('Failure')
 
   await handler(args)
 
-  expect(mockedHandleError).toBeCalledWith(expect.objectContaining({ message: 'oops' }))
+  expect(mockedHandleError).toBeCalledWith(expect.objectContaining({ message: 'Not all tests passed' }))
 })
-
-// TODO: desired behaviour
-it.todo('creates a type validator if there are types in the config')
-
-it.todo('does not create a type validator if there are no types in the config')

src/commands/test/handler.ts

@@ -1,8 +1,8 @@
 import { HandleError, CreateTypeValidator } from '~commands'
 import { NCDCLogger } from '~logger'
-import { TestConfigs } from './test'
+import { RunTests } from './test'
 import { createHttpClient } from './http-client'
-import { LoadConfig, LoadConfigStatus } from '~config/load'
+import { LoadConfig, LoadConfigStatus, GetTypeValidator } from '~config/load'
 import { ValidatedTestConfig, transformConfigs } from './config'
 import { red } from 'chalk'
 import { TypeValidator } from '~validation'
@@ -19,23 +19,20 @@ export const createHandler = (
   handleError: HandleError,
   createTypeValidator: CreateTypeValidator,
   logger: NCDCLogger,
-  testConfigs: TestConfigs,
+  runTests: RunTests,
   loadConfig: LoadConfig<ValidatedTestConfig>,
 ) => async (args: TestArgs): Promise<void> => {
   const { configPath, baseURL, tsconfigPath, schemaPath, force } = args
   if (!configPath) return handleError({ message: `configPath must be specified` })
   if (!baseURL) return handleError({ message: 'baseURL must be specified' })
 
   let typeValidator: TypeValidator | undefined
+  const getTypeValidator: GetTypeValidator = () => {
+    if (!typeValidator) typeValidator = createTypeValidator(tsconfigPath, force, schemaPath)
+    return typeValidator
+  }
 
-  const loadResult = await loadConfig(
-    configPath,
-    () => {
-      typeValidator = createTypeValidator(tsconfigPath, force, schemaPath)
-      return typeValidator
-    },
-    transformConfigs,
-  )
+  const loadResult = await loadConfig(configPath, getTypeValidator, transformConfigs)
 
   switch (loadResult.type) {
     case LoadConfigStatus.Success:
@@ -50,9 +47,12 @@ export const createHandler = (
       return handleError({ message: 'An unknown error ocurred' })
   }
 
-  try {
-    await testConfigs(baseURL, createHttpClient(baseURL), loadResult.configs, typeValidator)
-  } catch (err) {
-    return handleError(err)
-  }
+  const testResult = await runTests(
+    baseURL,
+    createHttpClient(baseURL),
+    loadResult.configs,
+    getTypeValidator,
+    logger,
+  )
+  if (testResult === 'Failure') return handleError({ message: 'Not all tests passed' })
 }