From 23d4c1ebc456f967e026d471bdb8ad418186cfc6 Mon Sep 17 00:00:00 2001 From: Peter Peterson Date: Wed, 11 Oct 2017 13:55:53 -0700 Subject: [PATCH] feat: refactor into libraries. add join flag. support multiple pr jobs BREAKING CHANGE: altered the signature for getWorkflow, getNextJobs --- README.md | 24 +++++- index.js | 119 +-------------------------- lib/getNextJobs.js | 33 ++++++++ lib/getWorkflow.js | 123 ++++++++++++++++++++++++++++ test/data/expected-output.json | 15 ++++ test/data/legacy-no-workflow.json | 7 ++ test/data/requires-workflow.json | 7 ++ test/index.test.js | 129 +----------------------------- test/lib/getNextJobs.test.js | 50 ++++++++++++ test/lib/getWorkflow.test.js | 97 ++++++++++++++++++++++ 10 files changed, 359 insertions(+), 245 deletions(-) create mode 100644 lib/getNextJobs.js create mode 100644 lib/getWorkflow.js create mode 100644 test/data/expected-output.json create mode 100644 test/data/legacy-no-workflow.json create mode 100644 test/data/requires-workflow.json create mode 100644 test/lib/getNextJobs.test.js create mode 100644 test/lib/getWorkflow.test.js diff --git a/README.md b/README.md index 95da859..3fe56f5 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,26 @@ npm install screwdriver-workflow-parser ``` +``` +const { getWorkflow, getNextJobs } = require('screwdriver-workflow-parser'); + +// Calculate the directed graph workflow from a pipeline config (and parse legacy workflows) +const workflowGraph = getWorkflow(pipelineConfig, { useLegacy: true }); + +/* +{ + nodes: [{ name: '~pr'}, { name: '~commit'}, { name: 'main' }], + edges: [{ src: '~pr', dest: 'main'}, { src: '~commit', dest: 'main'}] +} +*/ + +// Get a list of job names to start as a result of a commit event, e.g. [ 'a', 'b' ] +const commitJobsToTrigger = getNextJobs(workflowGraph, { trigger: '~commit' }); + +// Get a list of job names to start as a result of a pull-request event, e.g. [ 'PR-123:a' ] +const prJobsToTrigger = getNextJobs(workflowGraph, { trigger: '~pr', prNum: 123 }); +``` + ## Testing ```bash @@ -25,7 +45,7 @@ Code licensed under the BSD 3-Clause license. See LICENSE file for terms. [license-image]: https://img.shields.io/npm/l/screwdriver-workflow-parser.svg [issues-image]: https://img.shields.io/github/issues/screwdriver-cd/workflow-parser.svg [issues-url]: https://github.com/screwdriver-cd/workflow-parser/issues -[status-image]: https://cd.screwdriver.cd/pipelines/pipelineid/badge -[status-url]: https://cd.screwdriver.cd/pipelines/pipelineid +[status-image]: https://cd.screwdriver.cd/pipelines/352/badge +[status-url]: https://cd.screwdriver.cd/pipelines/352 [daviddm-image]: https://david-dm.org/screwdriver-cd/workflow-parser.svg?theme=shields.io [daviddm-url]: https://david-dm.org/screwdriver-cd/workflow-parser diff --git a/index.js b/index.js index 4adae0d..baa7f91 100644 --- a/index.js +++ b/index.js @@ -1,121 +1,6 @@ 'use strict'; -/** - * Get the list of nodes for the graph - * @method calculateNodes - * @param {Object} jobs Hash of job configs - * @return {Array} List of nodes (jobs) - */ -const calculateNodes = (jobs) => { - const nodes = []; - - Object.keys(jobs).forEach((j) => { - nodes.push({ name: j }); - }); - - return nodes; -}; - -/** - * Get all the edges of the directed graph from a legacy workflow config - * @method calculateLegacyEdges - * @param {Array} workflow List of all jobs in the workflow excluding "main" - * @return {Array} List of edge objects { src, dest } - */ -const calculateLegacyEdges = (workflow) => { - // In legacy-mode "main" is always required to exist, and be the target of commit and pr - const edges = [ - { src: '~pr', dest: 'main' }, - { src: '~commit', dest: 'main' } - ]; - - // Legacy-mode workflows are always linear, starting with main - let src = 'main'; - - workflow.forEach((dest) => { - edges.push({ src, dest }); - src = dest; - }); - - return edges; -}; - -/** - * Calculate edges of directed graph based on "requires" property of jobs - * @method calculateEdges - * @param {Object} jobs Hash of job configurations - * @return {Array} List of graph edges { src, dest } - */ -const calculateEdges = (jobs) => { - const edges = []; - - Object.keys(jobs).forEach((j) => { - const job = jobs[j]; - const dest = j; - - if (Array.isArray(job.requires)) { - job.requires.forEach((src) => { - edges.push({ src, dest }); - }); - } - }); - - return edges; -}; - -/** - * Given a pipeline config, return a directed graph configuration that describes the workflow - * @method getWorkflow - * @param {Object} obj - * @param {Object} obj.config A pipeline config - * @param {Object} obj.config.jobs Hash of job configs - * @param {Array} [obj.config.workflow] Legacy workflow config - * @param {Boolean} [obj.useLegacy] Flag to process legacy workflows - * @return {Object} List of nodes and edges { nodes, edges } - */ -const getWorkflow = ({ config: pipelineConfig, useLegacy = false }) => { - const jobConfig = pipelineConfig.jobs; - let edges = []; - - if (!jobConfig) { - throw new Error('No Job config provided'); - } - - const hasRequiresConfig = Object.keys(jobConfig) - .some(j => Array.isArray(jobConfig[j].requires)); - - if (useLegacy && !hasRequiresConfig) { - // Work out whether there is a user defined workflow, - // or if we use the order of jobs defined in jobConfig - const workflow = Array.isArray(pipelineConfig.workflow) ? - pipelineConfig.workflow : - Object.keys(jobConfig).filter(j => j !== 'main'); // remove main since that is a hard dependency - - edges = calculateLegacyEdges(workflow); - } else { - edges = calculateEdges(jobConfig); - } - - return { nodes: calculateNodes(jobConfig), edges }; -}; - -/** - * Calculate the next jobs to execute, given a workflow and a trigger job - * @method getNextJobs - * @param {Object} workflow Directed graph representation of workflow - * @param {String} trigger Name of event that triggers jobs (~pr, ~commit, JobName) - * @return {Array} List of job names - */ -const getNextJobs = (workflow, trigger) => { - const jobs = new Set(); - - workflow.edges.forEach((edge) => { - if (edge.src === trigger) { - jobs.add(edge.dest); - } - }); - - return Array.from(jobs); -}; +const getWorkflow = require('./lib/getWorkflow'); +const getNextJobs = require('./lib/getNextJobs'); module.exports = { getWorkflow, getNextJobs }; diff --git a/lib/getNextJobs.js b/lib/getNextJobs.js new file mode 100644 index 0000000..4ba3837 --- /dev/null +++ b/lib/getNextJobs.js @@ -0,0 +1,33 @@ +'use strict'; + +/** + * Calculate the next jobs to execute, given a workflow and a trigger job + * @method getNextJobs + * @param {Object} workflowGraph Directed graph representation of workflow + * @param {Object} config + * @param {String} config.trigger The triggering event (~pr, ~commit, jobName) + * @param {String} [config.prNum] The PR number (required when ~pr trigger) + * @return {Array} List of job names + */ +const getNextJobs = (workflowGraph, config) => { + const jobs = new Set(); + + if (!config || !config.trigger) { + throw new Error('Must provide a trigger'); + } + + if (config.trigger === '~pr' && !config.prNum) { + throw new Error('Must provide a PR number with "~pr" trigger'); + } + + workflowGraph.edges.forEach((edge) => { + if (edge.src === config.trigger) { + // Make PR jobs PR-$num:$cloneJob (not sure how to better handle multiple PR jobs) + jobs.add(config.trigger === '~pr' ? `PR-${config.prNum}:${edge.dest}` : edge.dest); + } + }); + + return Array.from(jobs); +}; + +module.exports = getNextJobs; diff --git a/lib/getWorkflow.js b/lib/getWorkflow.js new file mode 100644 index 0000000..520637a --- /dev/null +++ b/lib/getWorkflow.js @@ -0,0 +1,123 @@ +'use strict'; + +/** + * Get the list of nodes for the graph + * @method calculateNodes + * @param {Object} jobs Hash of job configs + * @return {Array} List of nodes (jobs) + */ +const calculateNodes = (jobs) => { + const nodes = [ + { name: '~pr' }, + { name: '~commit' } + ]; + + Object.keys(jobs).forEach((name) => { + nodes.push({ name }); + }); + + return nodes; +}; + +/** + * Get all the edges of the directed graph from a legacy workflow config + * @method calculateLegacyEdges + * @param {Array} workflow List of all jobs in the workflow excluding "main" + * @return {Array} List of edge objects { src, dest } + */ +const calculateLegacyEdges = (workflow) => { + // In legacy-mode "main" is always required to exist, and be the target of commit and pr + const edges = [ + { src: '~pr', dest: 'main' }, + { src: '~commit', dest: 'main' } + ]; + + // Legacy-mode workflows are always linear, starting with main + let src = 'main'; + + workflow.forEach((dest) => { + edges.push({ src, dest }); + src = dest; + }); + + return edges; +}; + +/** + * Calculate edges of directed graph based on "requires" property of jobs + * @method calculateEdges + * @param {Object} jobs Hash of job configurations + * @return {Array} List of graph edges { src, dest } + */ +const calculateEdges = (jobs) => { + const edges = []; + + Object.keys(jobs).forEach((j) => { + const job = jobs[j]; + const dest = j; + + if (Array.isArray(job.requires)) { + const specialTriggers = job.requires.filter(name => name.charAt(0) === '~'); + const normalTriggers = job.requires.filter(name => name.charAt(0) !== '~'); + const isJoin = normalTriggers.length > 1; + + specialTriggers.forEach((src) => { + edges.push({ src, dest }); + }); + + normalTriggers.forEach((src) => { + const obj = { src, dest }; + + if (isJoin) { + obj.join = true; + } + + edges.push(obj); + }); + } + }); + + return edges; +}; + +/** + * Given a pipeline config, return a directed graph configuration that describes the workflow + * @method getWorkflow + * @param {Object} pipelineConfig A Pipeline Config + * @param {Object} pipelineConfig.jobs Hash of job configs + * @param {Array} [pipelineConfig.workflow] Legacy workflow config + * @param {Object} [config] configuration object + * @param {Boolean} [config.useLegacy] Flag to process legacy workflows + * @return {Object} List of nodes and edges { nodes, edges } + */ +const getWorkflow = (pipelineConfig, config = { useLegacy: false }) => { + const jobConfig = pipelineConfig.jobs; + let edges = []; + + if (!jobConfig) { + throw new Error('No Job config provided'); + } + + const hasRequiresConfig = Object.keys(jobConfig) + .some(j => Array.isArray(jobConfig[j].requires)); + + if (config.useLegacy && !hasRequiresConfig) { + // Work out whether there is a user defined workflow, + // or if we use the order of jobs defined in jobConfig + let workflow = Array.isArray(pipelineConfig.workflow) ? + pipelineConfig.workflow : + Object.keys(jobConfig); + + // remove main since that is a hard dependency in legacy workflows + // main is already accounted for in calculateLegacyEdges + workflow = workflow.filter(j => j !== 'main'); + + edges = calculateLegacyEdges(workflow); + } else { + edges = calculateEdges(jobConfig); + } + + return { nodes: calculateNodes(jobConfig), edges }; +}; + +module.exports = getWorkflow; diff --git a/test/data/expected-output.json b/test/data/expected-output.json new file mode 100644 index 0000000..e36d608 --- /dev/null +++ b/test/data/expected-output.json @@ -0,0 +1,15 @@ +{ + "nodes": [ + { "name": "~pr" }, + { "name": "~commit" }, + { "name": "main" }, + { "name": "foo" }, + { "name": "bar" } + ], + "edges": [ + { "src": "~pr", "dest": "main" }, + { "src": "~commit", "dest": "main" }, + { "src": "main", "dest": "foo" }, + { "src": "foo", "dest": "bar" } + ] +} diff --git a/test/data/legacy-no-workflow.json b/test/data/legacy-no-workflow.json new file mode 100644 index 0000000..eed901b --- /dev/null +++ b/test/data/legacy-no-workflow.json @@ -0,0 +1,7 @@ +{ + "jobs": { + "main": {}, + "foo": {}, + "bar": {} + } +} diff --git a/test/data/requires-workflow.json b/test/data/requires-workflow.json new file mode 100644 index 0000000..3bc9cd0 --- /dev/null +++ b/test/data/requires-workflow.json @@ -0,0 +1,7 @@ +{ + "jobs": { + "main": { "requires": ["~pr", "~commit"] }, + "foo": { "requires": ["main"] }, + "bar": { "requires": ["foo"] } + } +} diff --git a/test/index.test.js b/test/index.test.js index 0213053..6852e01 100644 --- a/test/index.test.js +++ b/test/index.test.js @@ -3,132 +3,9 @@ const assert = require('chai').assert; const parser = require('../index'); -const LEGACY_NO_WORKFLOW = { - jobs: { - main: {}, - foo: {}, - bar: {} - } -}; -const LEGACY_WITH_WORKFLOW = Object.assign({}, LEGACY_NO_WORKFLOW); - -LEGACY_WITH_WORKFLOW.workflow = ['foo', 'bar']; - -const REQUIRES_WORKFLOW = { - jobs: { - main: { requires: ['~pr', '~commit'] }, - foo: { requires: ['main'] }, - bar: { requires: ['foo'] } - } -}; - -const LEGACY_AND_REQUIRES_WORKFLOW = Object.assign({}, REQUIRES_WORKFLOW); - -LEGACY_WITH_WORKFLOW.workflow = ['foo', 'bar']; - -const EXPECTED_OUTPUT = { - nodes: [ - { name: 'main' }, - { name: 'foo' }, - { name: 'bar' } - ], - edges: [ - { src: '~pr', dest: 'main' }, - { src: '~commit', dest: 'main' }, - { src: 'main', dest: 'foo' }, - { src: 'foo', dest: 'bar' } - ] -}; - -const NO_EDGES = Object.assign({}, EXPECTED_OUTPUT); - -NO_EDGES.edges = []; - describe('index test', () => { - describe('getWorkflow', () => { - it('should throw if it is not given correct input', () => { - assert.throws(() => parser.getWorkflow({ config: {} }), - Error, 'No Job config provided'); - }); - - it('should produce directed graph when legacy mode is on', () => { - assert.deepEqual(parser.getWorkflow({ - config: LEGACY_NO_WORKFLOW, - useLegacy: true - }), EXPECTED_OUTPUT, 'no legacy workflow defined'); - assert.deepEqual(parser.getWorkflow({ - config: LEGACY_WITH_WORKFLOW, - useLegacy: true - }), EXPECTED_OUTPUT, 'has legacy workflow defined'); - assert.deepEqual(parser.getWorkflow({ - config: REQUIRES_WORKFLOW, - useLegacy: true - }), EXPECTED_OUTPUT, 'requires-style workflow'); - assert.deepEqual(parser.getWorkflow({ - config: LEGACY_AND_REQUIRES_WORKFLOW, - useLegacy: true - }), EXPECTED_OUTPUT, 'both legacy and non-legacy workflows'); - }); - - it('should convert a legacy config to graph with no edges when legacy mode off', () => { - assert.deepEqual(parser.getWorkflow({ - config: LEGACY_WITH_WORKFLOW, - useLegacy: false - }), NO_EDGES, 'has legacy workflow defined'); - assert.deepEqual(parser.getWorkflow({ - config: LEGACY_NO_WORKFLOW, - useLegacy: false - }), NO_EDGES, 'legacy, no workflow defined'); - }); - - it('should convert a config with job-requires workflow to directed graph', () => { - assert.deepEqual(parser.getWorkflow({ - config: REQUIRES_WORKFLOW, - useLegacy: false - }), EXPECTED_OUTPUT, 'requires-style workflow'); - assert.deepEqual(parser.getWorkflow({ - config: LEGACY_AND_REQUIRES_WORKFLOW, - useLegacy: false - }), EXPECTED_OUTPUT, 'both legacy and non-legacy workflows'); - }); - - it('should handle detatched jobs', () => { - const result = parser.getWorkflow({ - config: { - jobs: { - foo: {}, - bar: { requires: ['foo'] } - } - } - }); - - assert.deepEqual(result, { - nodes: [{ name: 'foo' }, { name: 'bar' }], - edges: [{ src: 'foo', dest: 'bar' }] - }); - }); - }); - - describe('getNextJobs', () => { - it('should figure out what jobs start next', () => { - assert.deepEqual(parser.getNextJobs(EXPECTED_OUTPUT, '~pr'), ['main']); - assert.deepEqual(parser.getNextJobs(EXPECTED_OUTPUT, '~commit'), ['main']); - assert.deepEqual(parser.getNextJobs(EXPECTED_OUTPUT, 'main'), ['foo']); - assert.deepEqual(parser.getNextJobs(EXPECTED_OUTPUT, 'foo'), ['bar']); - assert.deepEqual(parser.getNextJobs(EXPECTED_OUTPUT, 'bar'), []); - assert.deepEqual(parser.getNextJobs(EXPECTED_OUTPUT, 'banana'), []); - - const parallelWorkflow = { - edges: [ - { src: 'a', dest: 'b' }, - { src: 'a', dest: 'c' }, - { src: 'a', dest: 'd' }, - { src: 'b', dest: 'e' } - ] - }; - - assert.deepEqual(parser.getNextJobs(parallelWorkflow, 'a'), ['b', 'c', 'd']); - assert.deepEqual(parser.getNextJobs(parallelWorkflow, 'b'), ['e']); - }); + it('should bundle all its libraries', () => { + assert.isFunction(parser.getWorkflow); + assert.isFunction(parser.getNextJobs); }); }); diff --git a/test/lib/getNextJobs.test.js b/test/lib/getNextJobs.test.js new file mode 100644 index 0000000..5e33c6c --- /dev/null +++ b/test/lib/getNextJobs.test.js @@ -0,0 +1,50 @@ +'use strict'; + +const assert = require('chai').assert; +const getNextJobs = require('../../lib/getNextJobs'); +const WORKFLOW = require('../data/expected-output'); + +describe('getNextJobs', () => { + it('should throw if trigger not provided', () => { + assert.throws(() => getNextJobs(WORKFLOW, {}), + Error, 'Must provide a trigger'); + }); + + it('should throw if prNum not provided for ~pr events', () => { + assert.throws(() => getNextJobs(WORKFLOW, { trigger: '~pr' }), + Error, 'Must provide a PR number with "~pr" trigger'); + }); + + it('should figure out what jobs start next', () => { + // trigger for a pr event + assert.deepEqual(getNextJobs(WORKFLOW, { + trigger: '~pr', + prNum: '123' + }), ['PR-123:main']); + // trigger for commit event + assert.deepEqual(getNextJobs(WORKFLOW, { trigger: '~commit' }), ['main']); + // trigger after job "main" + assert.deepEqual(getNextJobs(WORKFLOW, { trigger: 'main' }), ['foo']); + // trigger after job "foo" + assert.deepEqual(getNextJobs(WORKFLOW, { trigger: 'foo' }), ['bar']); + // trigger after job "bar"" + assert.deepEqual(getNextJobs(WORKFLOW, { trigger: 'bar' }), []); + // trigger after non-existing job "main" + assert.deepEqual(getNextJobs(WORKFLOW, { trigger: 'banana' }), []); + + const parallelWorkflow = { + edges: [ + { src: 'a', dest: 'b' }, + { src: 'a', dest: 'c' }, + { src: 'a', dest: 'd' }, + { src: 'b', dest: 'e' } + ] + }; + + // trigger multiple after job "a" + assert.deepEqual(getNextJobs(parallelWorkflow, { trigger: 'a' }), + ['b', 'c', 'd']); + // trigger one after job "b" + assert.deepEqual(getNextJobs(parallelWorkflow, { trigger: 'b' }), ['e']); + }); +}); diff --git a/test/lib/getWorkflow.test.js b/test/lib/getWorkflow.test.js new file mode 100644 index 0000000..80b9496 --- /dev/null +++ b/test/lib/getWorkflow.test.js @@ -0,0 +1,97 @@ +'use strict'; + +const assert = require('chai').assert; +const getWorkflow = require('../../lib/getWorkflow'); + +const LEGACY_NO_WORKFLOW = require('../data/legacy-no-workflow'); +const LEGACY_WITH_WORKFLOW = Object.assign({}, LEGACY_NO_WORKFLOW); + +LEGACY_WITH_WORKFLOW.workflow = ['foo', 'bar']; + +const REQUIRES_WORKFLOW = require('../data/requires-workflow'); +const LEGACY_AND_REQUIRES_WORKFLOW = Object.assign({}, REQUIRES_WORKFLOW); + +LEGACY_WITH_WORKFLOW.workflow = ['foo', 'bar']; + +const EXPECTED_OUTPUT = require('../data/expected-output'); +const NO_EDGES = Object.assign({}, EXPECTED_OUTPUT); + +NO_EDGES.edges = []; + +describe('getWorkflow', () => { + it('should throw if it is not given correct input', () => { + assert.throws(() => getWorkflow({ config: {} }), + Error, 'No Job config provided'); + }); + + it('should produce directed graph when legacy mode is on', () => { + assert.deepEqual(getWorkflow(LEGACY_NO_WORKFLOW, { + useLegacy: true + }), EXPECTED_OUTPUT, 'no legacy workflow defined'); + assert.deepEqual(getWorkflow(LEGACY_WITH_WORKFLOW, { + useLegacy: true + }), EXPECTED_OUTPUT, 'has legacy workflow defined'); + assert.deepEqual(getWorkflow(REQUIRES_WORKFLOW, { + useLegacy: true + }), EXPECTED_OUTPUT, 'requires-style workflow'); + assert.deepEqual(getWorkflow(LEGACY_AND_REQUIRES_WORKFLOW, { + useLegacy: true + }), EXPECTED_OUTPUT, 'both legacy and non-legacy workflows'); + }); + + it('should convert a legacy config to graph with no edges when legacy mode off', () => { + assert.deepEqual(getWorkflow(LEGACY_WITH_WORKFLOW), + NO_EDGES, 'has legacy workflow defined'); + assert.deepEqual(getWorkflow(LEGACY_NO_WORKFLOW), + NO_EDGES, 'legacy, no workflow defined'); + }); + + it('should convert a config with job-requires workflow to directed graph', () => { + assert.deepEqual(getWorkflow(REQUIRES_WORKFLOW), + EXPECTED_OUTPUT, 'requires-style workflow'); + assert.deepEqual(getWorkflow(LEGACY_AND_REQUIRES_WORKFLOW), + EXPECTED_OUTPUT, 'both legacy and non-legacy workflows'); + }); + + it('should handle detatched jobs', () => { + const result = getWorkflow({ + jobs: { + foo: {}, + bar: { requires: ['foo'] } + } + }); + + assert.deepEqual(result, { + nodes: [{ name: '~pr' }, { name: '~commit' }, { name: 'foo' }, { name: 'bar' }], + edges: [{ src: 'foo', dest: 'bar' }] + }); + }); + + it('should handle joins', () => { + const result = getWorkflow({ + jobs: { + foo: { }, + bar: { requires: ['foo'] }, + baz: { requires: ['foo'] }, + bax: { requires: ['bar', 'baz'] } + } + }); + + assert.deepEqual(result, { + nodes: [ + { name: '~pr' }, + { name: '~commit' }, + { name: 'foo' }, + { name: 'bar' }, + { name: 'baz' }, + { name: 'bax' } + ], + edges: [ + { src: 'foo', dest: 'bar' }, + { src: 'foo', dest: 'baz' }, + { src: 'bar', dest: 'bax', join: true }, + { src: 'baz', dest: 'bax', join: true } + ] + }); + }); +});