diff --git a/integration-tests/execute/CHANGELOG.md b/integration-tests/execute/CHANGELOG.md new file mode 100644 index 000000000..25a096473 --- /dev/null +++ b/integration-tests/execute/CHANGELOG.md @@ -0,0 +1,10 @@ +# @openfn/integration-tests-execute + +## 1.0.1 + +### Patch Changes + +- Updated dependencies [40fd45b] +- Updated dependencies [40fd45b] + - @openfn/compiler@0.2.0 + - @openfn/runtime@1.4.1 diff --git a/integration-tests/execute/package.json b/integration-tests/execute/package.json new file mode 100644 index 000000000..ed4bf550a --- /dev/null +++ b/integration-tests/execute/package.json @@ -0,0 +1,32 @@ +{ + "name": "@openfn/integration-tests-execute", + "private": true, + "version": "1.0.1", + "description": "Job execution tests", + "author": "Open Function Group ", + "license": "ISC", + "type": "module", + "scripts": { + "test": "pnpm ava" + }, + "dependencies": { + "@openfn/compiler": "workspace:^", + "@openfn/language-common": "1.7.7", + "@openfn/language-http": "6.4.0", + "@openfn/runtime": "workspace:^", + "@types/node": "^18.15.13", + "ava": "5.3.1", + "date-fns": "^2.30.0", + "rimraf": "^3.0.2", + "ts-node": "10.8.1", + "tslib": "^2.4.0", + "typescript": "^5.1.6" + }, + "files": [ + "dist", + "README.md" + ], + "devDependencies": { + "@types/rimraf": "^3.0.2" + } +} diff --git a/integration-tests/execute/readme.md b/integration-tests/execute/readme.md new file mode 100644 index 000000000..ce9813182 --- /dev/null +++ b/integration-tests/execute/readme.md @@ -0,0 +1,5 @@ +This is a suite of examples of jobs. + +We don't really have a place where we can just write and test arbtirary job code with compilation. + +You can do it through the CLI or worker but they have significant overheads. diff --git a/integration-tests/execute/src/execute.ts b/integration-tests/execute/src/execute.ts new file mode 100644 index 000000000..49aba31d7 --- /dev/null +++ b/integration-tests/execute/src/execute.ts @@ -0,0 +1,35 @@ +import path from 'node:path'; +import run from '@openfn/runtime'; +import compiler from '@openfn/compiler'; + +const execute = async (job: string, state: any, adaptor = 'common') => { + // compile with common and dumb imports + const options = { + 'add-imports': { + adaptor: { + name: `@openfn/language-${adaptor}`, + exportAll: true, + }, + }, + }; + const compiled = compiler(job, options); + // console.log(compiled); + + const result = await run(compiled, state, { + // preload the linker with some locally installed modules + linker: { + modules: { + '@openfn/language-common': { + path: path.resolve('node_modules/@openfn/language-common'), + }, + '@openfn/language-http': { + path: path.resolve('node_modules/@openfn/language-http'), + }, + }, + }, + }); + + return result; +}; + +export default execute; diff --git a/integration-tests/execute/src/index.ts b/integration-tests/execute/src/index.ts new file mode 100644 index 000000000..6c7d29fe5 --- /dev/null +++ b/integration-tests/execute/src/index.ts @@ -0,0 +1,3 @@ +import execute from './execute'; + +export default execute; diff --git a/integration-tests/execute/test/execute.test.ts b/integration-tests/execute/test/execute.test.ts new file mode 100644 index 000000000..24e48e67c --- /dev/null +++ b/integration-tests/execute/test/execute.test.ts @@ -0,0 +1,148 @@ +import test from 'ava'; + +import execute from '../src/execute'; + +const wait = `function wait() { + return (state) => + new Promise((resolve) => { + setTimeout(() => resolve(state), 2); + }); +};`; + +test.serial('should return state', async (t) => { + const state = { data: { x: 1 } }; + + const job = ` + fn(s => s) + `; + const result = await execute(job, state); + + t.deepEqual(state, result); +}); + +test.serial('should use .then()', async (t) => { + const state = { data: { x: 1 } }; + + const job = ` + fn(s => s) + .then((s) => + ({ + data: { x: 33 } + }) + ) + `; + const result = await execute(job, state); + + t.deepEqual(result, { data: { x: 33 } }); +}); + +test.serial('should chain .then() with state', async (t) => { + const state = { data: { x: 1 } }; + + const job = ` + fn(s => ({ x: 1 })) + .then((s) => + ({ + x: s.x + 1 + }) + ) + `; + const result = await execute(job, state); + + t.deepEqual(result, { x: 2 }); +}); + +test.serial('should use .then() as an argument', async (t) => { + const state = {}; + + const job = `fn( + fn(() => ({ x: 5 })).then((s) => ({ x: s.x + 1})) + )`; + const result = await execute(job, state); + + t.deepEqual(result, { x: 6 }); +}); + +test.serial('use then() with wait()', async (t) => { + const state = { + data: { + x: 22, + }, + }; + + const job = `${wait} + wait().then(fn(s => s))`; + + const result = await execute(job, state); + + t.deepEqual(result.data, { x: 22 }); +}); + +test.serial('catch an error and return it', async (t) => { + const state = { + data: { + x: 22, + }, + }; + + const job = `fn(() => { + throw { err: true } + }).catch(e => e)`; + + const result = await execute(job, state); + t.deepEqual(result, { err: true }); +}); + +test.serial('catch an error and re-throw it', async (t) => { + const state = { + data: { + x: 22, + }, + }; + + const job = `fn(() => { + throw new Error('err') + }).catch(e => { throw e })`; + + const result = await execute(job, state); + t.is(result.errors['job-1'].name, 'JobError'); + t.is(result.errors['job-1'].message, 'err'); +}); + +test.serial('catch an error and return state', async (t) => { + const state = { + data: { + x: 22, + }, + }; + + const job = `fn(() => { + throw { err: true } + }).catch((e, s) => s)`; + + const result = await execute(job, state); + t.deepEqual(result, state); +}); + +test.serial('each with then ', async (t) => { + const state = { + ids: [1, 2, 3], + results: [], + }; + + const job = `each($.ids, + get(\`https://jsonplaceholder.typicode.com/todos/\${$.data}\`).then( + (s) => { + s.results.push(s.data); + return s; + } + ) + )`; + + const result = await execute(job, state, 'http'); + + t.is(result.results.length, 3); + t.is(result.results[0].id, 1); + t.is(result.results[1].id, 2); + t.is(result.results[2].id, 3); +}); diff --git a/integration-tests/execute/tsconfig.json b/integration-tests/execute/tsconfig.json new file mode 100644 index 000000000..9bffa80cb --- /dev/null +++ b/integration-tests/execute/tsconfig.json @@ -0,0 +1,14 @@ +{ + "ts-node": { + "experimentalSpecifierResolution": "node" + }, + "compilerOptions": { + "allowSyntheticDefaultImports": true, + "module": "es2020", + "moduleResolution": "node", + "allowJs": true, + "isolatedModules": true, + "noEmit": true, + "skipLibCheck": true + } +} diff --git a/integration-tests/worker/CHANGELOG.md b/integration-tests/worker/CHANGELOG.md index 26c43d603..bcfc5a864 100644 --- a/integration-tests/worker/CHANGELOG.md +++ b/integration-tests/worker/CHANGELOG.md @@ -1,5 +1,14 @@ # @openfn/integration-tests-worker +## 1.0.51 + +### Patch Changes + +- Updated dependencies + - @openfn/engine-multi@1.2.0 + - @openfn/ws-worker@1.4.0 + - @openfn/lightning-mock@2.0.14 + ## 1.0.50 ### Patch Changes diff --git a/integration-tests/worker/package.json b/integration-tests/worker/package.json index f14e63873..78e871557 100644 --- a/integration-tests/worker/package.json +++ b/integration-tests/worker/package.json @@ -1,7 +1,7 @@ { "name": "@openfn/integration-tests-worker", "private": true, - "version": "1.0.50", + "version": "1.0.51", "description": "Lightning WOrker integration tests", "author": "Open Function Group ", "license": "ISC", diff --git a/packages/cli/CHANGELOG.md b/packages/cli/CHANGELOG.md index 9c2f8c72e..5f725a5fc 100644 --- a/packages/cli/CHANGELOG.md +++ b/packages/cli/CHANGELOG.md @@ -1,5 +1,18 @@ # @openfn/cli +## 1.7.0 + +### Minor Changes + +- Allow operations to behave like promises (ie, support fn().then()) + +### Patch Changes + +- Updated dependencies [40fd45b] +- Updated dependencies [40fd45b] + - @openfn/compiler@0.2.0 + - @openfn/runtime@1.4.1 + ## 1.6.1 ### Patch Changes diff --git a/packages/cli/package.json b/packages/cli/package.json index f724569b3..748b95268 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@openfn/cli", - "version": "1.6.1", + "version": "1.7.0", "description": "CLI devtools for the openfn toolchain.", "engines": { "node": ">=18", diff --git a/packages/compiler/CHANGELOG.md b/packages/compiler/CHANGELOG.md index bf050b324..fbe2f4d99 100644 --- a/packages/compiler/CHANGELOG.md +++ b/packages/compiler/CHANGELOG.md @@ -1,5 +1,12 @@ # @openfn/compiler +## 0.2.0 + +### Minor Changes + +- 40fd45b: Add promises transformer + Don't try and import variables declared in other import statements + ## 0.1.4 ### Patch Changes diff --git a/packages/compiler/package.json b/packages/compiler/package.json index b40de120f..9bca4763f 100644 --- a/packages/compiler/package.json +++ b/packages/compiler/package.json @@ -1,6 +1,6 @@ { "name": "@openfn/compiler", - "version": "0.1.4", + "version": "0.2.0", "description": "Compiler and language tooling for openfn jobs.", "author": "Open Function Group ", "license": "ISC", diff --git a/packages/compiler/src/transform.ts b/packages/compiler/src/transform.ts index 06f90195b..ec041fd49 100644 --- a/packages/compiler/src/transform.ts +++ b/packages/compiler/src/transform.ts @@ -6,6 +6,7 @@ import createLogger, { Logger } from '@openfn/logger'; import addImports, { AddImportsOptions } from './transforms/add-imports'; import ensureExports from './transforms/ensure-exports'; import lazyState from './transforms/lazy-state'; +import promises from './transforms/promises'; import topLevelOps, { TopLevelOpsOptions, } from './transforms/top-level-operations'; @@ -38,6 +39,7 @@ export type TransformOptions = { ['top-level-operations']?: TopLevelOpsOptions | boolean; ['test']?: any; ['lazy-state']?: any; + ['promises']?: any; }; const defaultLogger = createLogger(); @@ -50,6 +52,7 @@ export default function transform( if (!transformers) { transformers = [ lazyState, + promises, ensureExports, topLevelOps, addImports, diff --git a/packages/compiler/src/transforms/add-imports.ts b/packages/compiler/src/transforms/add-imports.ts index 5a4a268db..d6c9ae36c 100644 --- a/packages/compiler/src/transforms/add-imports.ts +++ b/packages/compiler/src/transforms/add-imports.ts @@ -108,6 +108,11 @@ export function findAllDanglingIdentifiers(ast: ASTNode) { const result: IdentifierList = {}; visit(ast, { visitIdentifier: function (path) { + // If this is part of an import statement, do nothing + if (n.ImportSpecifier.check(path.parent.node)) { + return false; + } + // undefined and NaN are treated as a regular identifier if (path.node.name === 'undefined' || path.node.name === 'NaN') { return false; diff --git a/packages/compiler/src/transforms/promises.ts b/packages/compiler/src/transforms/promises.ts new file mode 100644 index 000000000..355c3e503 --- /dev/null +++ b/packages/compiler/src/transforms/promises.ts @@ -0,0 +1,173 @@ +import { namedTypes as n, builders as b } from 'ast-types'; + +import type { NodePath } from 'ast-types/lib/node-path'; + +const NO_DEFER_DECLARATION_ERROR = 'No defer declaration found'; + +export const assertDeferDeclaration = ( + program: NodePath | n.Program +) => { + if ((program as NodePath).node) { + program = (program as NodePath).node; + } + const p = program as n.Program; + for (const node of p.body) { + if (n.ImportDeclaration.check(node)) { + if (node.source.value === '@openfn/runtime') { + return true; + } + } + } + + throw new Error(NO_DEFER_DECLARATION_ERROR); +}; + +const injectDeferImport = (root: NodePath) => { + try { + assertDeferDeclaration(root); + } catch (e: any) { + if (e.message === NO_DEFER_DECLARATION_ERROR) { + const i = b.importDeclaration( + [b.importSpecifier(b.identifier('defer'), b.identifier('_defer'))], + b.stringLiteral('@openfn/runtime') + ); + + // Find the first non-import node and + let idx = 0; + for (const node of root.node.body) { + if (!n.ImportDeclaration.check(node)) { + break; + } + idx++; + } + root.node.body.splice(idx, 0, i); + } + } +}; + +/* + This function will replace a promise chain of the form op().then().then() + with a defer() function call, which breaks the operation and promise chain + into two parts, like this: + + defer(op(), p => p.then().then()) +*/ +export const rebuildPromiseChain = (expr: NodePath) => { + // We've just been handed something like looks like an operation with a promise chain + // ie, op().then().then() + // Walk down the call expression tree until we find the operation that's originally called + let op: NodePath | null = null; + let next = expr; + while (next) { + if (n.Identifier.check(next.node.callee)) { + op = next; + break; + } + if ( + n.MemberExpression.check(next.node.callee) && + !(next.node.callee.property as any).name?.match(/^(then|catch)$/) + ) { + op = next; + break; + } else { + next = next.get('callee', 'object'); + } + } + + if (!op) { + // If somehow we can't find the underling operation, abort + return; + } + + // Build the arguments to the defer() array + const deferArgs: any[] = [op.node]; + let catchFn; + + if (op.parent.node.property?.name === 'catch') { + // If there's a catch adjacent to the operation, we need to handle that a bit differently + catchFn = op.parent.parent.get('arguments', 0); + } + + // In the promise chain, replace the operation call with `p`, a promise + op.replace(b.identifier('p')); + + // Now we re-build the promise chain + // This is a bit different if the operation has a catch against it + if (catchFn) { + // remove the catch from the tree + const parent = catchFn.parent.parent; + + // if this catch is part of a longer chain, + // cut the catch out of the chain and replace it with p + if (parent.node.object === catchFn.parent.node) { + parent.get('object').replace(b.identifier('p')); + const chain = b.arrowFunctionExpression([b.identifier('p')], expr.node); + deferArgs.push(chain); + } else { + // Otherwise, if there is no then chain, just pass undefined + deferArgs.push(b.identifier('undefined')); + } + deferArgs.push(catchFn.node); + } else { + // If there's no catch, reparent the entire promise chian into an arrow + // ie, (p) => p.then().then() + const chain = b.arrowFunctionExpression([b.identifier('p')], expr.node); + if (chain) { + deferArgs.push(chain); + } + } + + // Finally, build and return the defer function call + const defer = b.callExpression(b.identifier('_defer'), deferArgs); + + expr.replace(defer); + + return defer; +}; + +const isTopScope = (path: NodePath) => { + let parent = path.parent; + while (parent) { + if (n.Program.check(parent.node)) { + return true; + } + if ( + n.ArrowFunctionExpression.check(parent.node) || + n.FunctionDeclaration.check(parent.node) || + n.FunctionExpression.check(parent.node) || + n.BlockStatement.check(parent.node) + // TODO more? + ) { + return false; + } + parent = parent.parent; + } + return true; +}; + +const visitor = (path: NodePath) => { + let root: NodePath = path; + while (!n.Program.check(root.node)) { + root = root.parent; + } + + // any Call expression with then|catch which is not in a nested scope + if ( + (path.node.callee as any).property?.name?.match(/^(then|catch)$/) && + isTopScope(path) + ) { + injectDeferImport(root); + rebuildPromiseChain(path); + + // do not traverse this tree any further + return true; + } +}; + +export default { + id: 'promises', + types: ['CallExpression'], + visitor, + // this should run before top-level operations are moved into the exports array + order: 0, +} as Transformer; diff --git a/packages/compiler/test/compile.test.ts b/packages/compiler/test/compile.test.ts index 41924d849..376902f64 100644 --- a/packages/compiler/test/compile.test.ts +++ b/packages/compiler/test/compile.test.ts @@ -7,35 +7,35 @@ test('ensure default exports is created', (t) => { const source = ''; const expected = 'export default [];'; const result = compile(source); - t.assert(result === expected); + t.is(result, expected); }); test('do not add default exports if exports exist', (t) => { const source = 'export const x = 10;'; const expected = 'export const x = 10;'; const result = compile(source); - t.assert(result === expected); + t.is(result, expected); }); test('compile a single operation', (t) => { const source = 'fn();'; const expected = 'export default [fn()];'; const result = compile(source); - t.assert(result === expected); + t.is(result, expected); }); test('compile a single operation without being fussy about semicolons', (t) => { const source = 'fn()'; const expected = 'export default [fn()];'; const result = compile(source); - t.assert(result === expected); + t.is(result, expected); }); test('compile multiple operations', (t) => { const source = 'fn();fn();fn();'; const expected = 'export default [fn(), fn(), fn()];'; const result = compile(source); - t.assert(result === expected); + t.is(result, expected); }); test('add imports', (t) => { @@ -50,7 +50,7 @@ test('add imports', (t) => { const source = 'fn();'; const expected = `import { fn } from "@openfn/language-common";\nexport default [fn()];`; const result = compile(source, options); - t.assert(result === expected); + t.is(result, expected); }); test('do not add imports', (t) => { @@ -66,7 +66,7 @@ test('do not add imports', (t) => { const source = "import { fn } from '@openfn/language-common'; fn();"; const expected = `import { fn } from '@openfn/language-common';\nexport default [fn()];`; const result = compile(source, options); - t.assert(result === expected); + t.is(result, expected); }); test('dumbly add imports', (t) => { @@ -81,7 +81,7 @@ test('dumbly add imports', (t) => { const source = "import { jam } from '@openfn/language-common'; jam(state);"; const expected = `import { jam } from '@openfn/language-common';\nexport default [jam(state)];`; const result = compile(source, options); - t.assert(result === expected); + t.is(result, expected); }); test('add imports with export all', (t) => { @@ -97,7 +97,7 @@ test('add imports with export all', (t) => { const source = 'fn();'; const expected = `import { fn } from "@openfn/language-common";\nexport * from "@openfn/language-common";\nexport default [fn()];`; const result = compile(source, options); - t.assert(result === expected); + t.is(result, expected); }); test('twitter example', async (t) => { @@ -119,24 +119,23 @@ test('compile with optional chaining', (t) => { const source = 'fn(a.b?.c);'; const expected = 'export default [fn(a.b?.c)];'; const result = compile(source); - t.assert(result === expected); + t.is(result, expected); }); test('compile with nullish coalescence', (t) => { const source = 'fn(a ?? b);'; const expected = 'export default [fn(a ?? b)];'; const result = compile(source); - t.assert(result === expected); + t.is(result, expected); }); test('compile a lazy state ($) expression', (t) => { const source = 'get($.data.endpoint);'; const expected = 'export default [get(state => state.data.endpoint)];'; const result = compile(source); - t.assert(result === expected); + t.is(result, expected); }); - test('compile a lazy state ($) expression with dumb imports', (t) => { const options = { 'add-imports': { @@ -149,8 +148,40 @@ test('compile a lazy state ($) expression with dumb imports', (t) => { const source = 'get($.data.endpoint);'; const expected = `import { get } from "@openfn/language-common"; export * from "@openfn/language-common"; -export default [get(state => state.data.endpoint)];` +export default [get(state => state.data.endpoint)];`; const result = compile(source, options); - t.assert(result === expected); -}); \ No newline at end of file + t.is(result, expected); +}); + +test('compile simple promise chain', (t) => { + const source = + 'get($.data.endpoint).then((s => { console.log(s.data); return state;} ));'; + + const expected = `import { defer as _defer } from "@openfn/runtime"; + +export default [_defer( + get(state => state.data.endpoint), + p => p.then((s => { console.log(s.data); return state;} )) +)];`; + + const result = compile(source); + t.is(result, expected); +}); + +test('compile simple promise chain with each', (t) => { + const source = `each( + "$.data[*]", + post("/upsert", (state) => state.data).then((s) => s) +)`; + + const expected = `import { defer as _defer } from "@openfn/runtime"; + +export default [each( + "$.data[*]", + _defer(post("/upsert", (state) => state.data), p => p.then((s) => s)) +)];`; + + const result = compile(source); + t.is(result, expected); +}); diff --git a/packages/compiler/test/transforms/add-imports.test.ts b/packages/compiler/test/transforms/add-imports.test.ts index f3d01cf4c..76eccffef 100644 --- a/packages/compiler/test/transforms/add-imports.test.ts +++ b/packages/compiler/test/transforms/add-imports.test.ts @@ -447,6 +447,34 @@ test("Don't add imports for ignored identifiers", async (t) => { t.assert(imports[0].imported.name === 'y'); }); +test("Don't add imports from import specifiers", async (t) => { + const ast = b.program([ + b.importDeclaration( + [ + b.importSpecifier(b.identifier('x')), + b.importSpecifier(b.identifier('y'), b.identifier('_y')), + ], + b.stringLiteral('@openfn/runtime') + ), + ]); + + const options = { + 'add-imports': { + adaptor: { + name: 'test-adaptor', + exports: [], + }, + }, + }; + + const transformed = transform(ast, [addImports], options) as n.Program; + + t.assert(transformed.body.length === 1); + const [first] = transformed.body; + t.assert(n.ImportDeclaration.check(first)); + t.assert(first.source.value === '@openfn/runtime'); +}); + test('export everything from an adaptor', (t) => { const ast = b.program([b.expressionStatement(b.identifier('x'))]); diff --git a/packages/compiler/test/transforms/promises.test.ts b/packages/compiler/test/transforms/promises.test.ts new file mode 100644 index 000000000..11049e3e1 --- /dev/null +++ b/packages/compiler/test/transforms/promises.test.ts @@ -0,0 +1,251 @@ +import test from 'ava'; +import { print } from 'recast'; +import { NodePath, namedTypes as n } from 'ast-types'; + +import promises, { + assertDeferDeclaration, + rebuildPromiseChain, +} from '../../src/transforms/promises'; +import parse from '../../src/parse'; +import transform from '../../src/transform'; + +test("assertDeferDeclaration: find defer if it's the only thing", (t) => { + const source = 'import { defer } from "@openfn/runtime"'; + + const ast = parse(source); + assertDeferDeclaration(ast.program); + t.pass('defer found'); +}); + +test('assertDeferDeclaration: throw if no defer import found', (t) => { + const source = 'import { defer } from "@openfn/common"'; + + const ast = parse(source); + try { + assertDeferDeclaration(ast.program); + } catch (e) { + t.pass('assertion correctly failed'); + } +}); + +test('assertDeferDeclaration: find defer among several statements', (t) => { + const source = `if(true) {}; + const d = () => {}; + import { defer } from "@openfn/runtime"; + function $defer () {}; + const _defer = false`; + + const ast = parse(source); + assertDeferDeclaration(ast.program); + t.pass('defer found'); +}); + +test('wrapFn: fn().then()', async (t) => { + const source = `fn(x).then(() => {})`; + const result = `_defer(fn(x), p => p.then(() => {}))`; + + const ast = parse(source); + const nodepath = new NodePath(ast.program); + const transformed = rebuildPromiseChain( + nodepath.get('body', 0, 'expression') + ); + + const { code } = print(transformed); + + t.log(code); + t.is(code, result); +}); + +test('wrapFn: fn.catch()', async (t) => { + const source = `fn(x).catch((e) => e)`; + const result = `_defer(fn(x), undefined, (e) => e)`; + + const ast = parse(source); + const nodepath = new NodePath(ast.program); + const transformed = rebuildPromiseChain( + nodepath.get('body', 0, 'expression') + ); + + const { code } = print(transformed); + + t.log(code); + t.is(code, result); +}); + +test('wrapFn: fn.then().then()', async (t) => { + const source = `fn(x).then((e) => e).then((e) => e)`; + const result = `_defer(fn(x), p => p.then((e) => e).then((e) => e))`; + + const ast = parse(source); + const nodepath = new NodePath(ast.program); + const transformed = rebuildPromiseChain( + nodepath.get('body', 0, 'expression') + ); + + const { code } = print(transformed); + + t.log(code); + t.is(code, result); +}); + +test('wrapFn: fn.catch().then()', async (t) => { + const source = `fn(x).catch((e) => e).then((s) => s)`; + const result = `_defer(fn(x), p => p.then((s) => s), (e) => e)`; + + const ast = parse(source); + const nodepath = new NodePath(ast.program); + const transformed = rebuildPromiseChain( + nodepath.get('body', 0, 'expression') + ); + + const { code } = print(transformed); + + t.log(code); + t.is(code, result); +}); + +test('wrapFn: fn.catch().then().then()', async (t) => { + const source = `fn(x).catch((e) => e).then((s) => s).then(s => s)`; + const result = `_defer(fn(x), p => p.then((s) => s).then(s => s), (e) => e)`; + + const ast = parse(source); + const nodepath = new NodePath(ast.program); + const transformed = rebuildPromiseChain( + nodepath.get('body', 0, 'expression') + ); + + const { code } = print(transformed); + + t.log(code); + t.is(code, result); +}); + +test('wrapFn: fn.catch().then().catch', async (t) => { + const source = `fn(x).catch((e) => e).then((s) => s).catch(e => e)`; + const result = `_defer(fn(x), p => p.then((s) => s).catch(e => e), (e) => e)`; + + const ast = parse(source); + const nodepath = new NodePath(ast.program); + const transformed = rebuildPromiseChain( + nodepath.get('body', 0, 'expression') + ); + + const { code } = print(transformed); + + t.log(code); + t.is(code, result); +}); + +test('transform: fn().then()', (t) => { + const source = `fn(x).then(s => s);`; + const result = `_defer(fn(x), p => p.then(s => s));`; + + const ast = parse(source); + + const transformed = transform(ast, [promises]) as n.File; + + assertDeferDeclaration(transformed.program); + + const { code } = print(transformed); + + t.log(code); + + const { code: transformedExport } = print(transformed.program.body.at(-1)); + t.is(transformedExport, result); +}); + +test('transform: fn().catch()', (t) => { + const source = `fn(x).catch(s => s);`; + const result = `_defer(fn(x), undefined, s => s);`; + + const ast = parse(source); + + const transformed = transform(ast, [promises]) as n.File; + + assertDeferDeclaration(transformed.program); + + const { code } = print(transformed); + t.log(code); + + const { code: transformedExport } = print(transformed.program.body.at(-1)); + t.is(transformedExport, result); +}); + +test('transform: only import once ', (t) => { + const source = `fn(x).then(s => s); +fn(x).then(s => s);`; + + const result = `import { defer as _defer } from "@openfn/runtime"; +_defer(fn(x), p => p.then(s => s)); +_defer(fn(x), p => p.then(s => s));`; + + const ast = parse(source); + + const transformed = transform(ast, [promises]) as n.File; + const { code } = print(transformed); + t.is(code, result); +}); + +test('transform: insert new import at the end of existing imports ', (t) => { + const source = `import x from 'y'; +fn(x).then(s => s);`; + + const result = `import x from 'y'; +import { defer as _defer } from "@openfn/runtime"; +_defer(fn(x), p => p.then(s => s));`; + + const ast = parse(source); + const transformed = transform(ast, [promises]) as n.File; + const { code } = print(transformed); + + t.is(code, result); +}); + +test('transform: fn(get().then())', (t) => { + const source = `fn(get(x).then(s => s));`; + const result = `fn(_defer(get(x), p => p.then(s => s)));`; + + const ast = parse(source); + + const transformed = transform(ast, [promises]) as n.File; + + assertDeferDeclaration(transformed.program); + + const { code } = print(transformed); + + t.log(code); + + const { code: transformedExport } = print(transformed.program.body.at(-1)); + t.is(transformedExport, result); +}); + +test('transform: fn(get().then(), get().then())', (t) => { + const source = `fn(get(x).then(s => s), post(x).then(s => s));`; + const result = `fn(_defer(get(x), p => p.then(s => s)), _defer(post(x), p => p.then(s => s)));`; + + const ast = parse(source); + + const transformed = transform(ast, [promises]) as n.File; + + assertDeferDeclaration(transformed.program); + + const { code } = print(transformed); + + t.log(code); + + const { code: transformedExport } = print(transformed.program.body.at(-1)); + t.is(transformedExport, result); +}); + +test('transform: ignore promises in a callback', (t) => { + const source = `fn((state) => { + return get().then((s) => s) +});`; + + const ast = parse(source); + + const transformed = transform(ast, [promises]) as n.File; + + const { code } = print(transformed); + t.is(code, source); +}); diff --git a/packages/engine-multi/CHANGELOG.md b/packages/engine-multi/CHANGELOG.md index 2c7b3b64e..98fd0c5b3 100644 --- a/packages/engine-multi/CHANGELOG.md +++ b/packages/engine-multi/CHANGELOG.md @@ -1,5 +1,18 @@ # engine-multi +## 1.2.0 + +### Minor Changes + +- Allow operations to behave like promises (ie, support fn().then()) + +### Patch Changes + +- Updated dependencies [40fd45b] +- Updated dependencies [40fd45b] + - @openfn/compiler@0.2.0 + - @openfn/runtime@1.4.1 + ## 1.1.13 ### Patch Changes diff --git a/packages/engine-multi/package.json b/packages/engine-multi/package.json index 732fd6dd3..d4b6045ff 100644 --- a/packages/engine-multi/package.json +++ b/packages/engine-multi/package.json @@ -1,6 +1,6 @@ { "name": "@openfn/engine-multi", - "version": "1.1.13", + "version": "1.2.0", "description": "Multi-process runtime engine", "main": "dist/index.js", "type": "module", diff --git a/packages/lightning-mock/CHANGELOG.md b/packages/lightning-mock/CHANGELOG.md index 50e82f280..4d97a24f4 100644 --- a/packages/lightning-mock/CHANGELOG.md +++ b/packages/lightning-mock/CHANGELOG.md @@ -1,5 +1,14 @@ # @openfn/lightning-mock +## 2.0.14 + +### Patch Changes + +- Updated dependencies [40fd45b] +- Updated dependencies + - @openfn/runtime@1.4.1 + - @openfn/engine-multi@1.2.0 + ## 2.0.13 ### Patch Changes diff --git a/packages/lightning-mock/package.json b/packages/lightning-mock/package.json index 2927b341d..2de385a0e 100644 --- a/packages/lightning-mock/package.json +++ b/packages/lightning-mock/package.json @@ -1,6 +1,6 @@ { "name": "@openfn/lightning-mock", - "version": "2.0.13", + "version": "2.0.14", "private": true, "description": "A mock Lightning server", "main": "dist/index.js", diff --git a/packages/runtime/CHANGELOG.md b/packages/runtime/CHANGELOG.md index 300940a2e..227b3cbf2 100644 --- a/packages/runtime/CHANGELOG.md +++ b/packages/runtime/CHANGELOG.md @@ -1,5 +1,11 @@ # @openfn/runtime +## 1.4.1 + +### Patch Changes + +- 40fd45b: Allow the linker to directly import some whitelisted packages + ## 1.4.0 ### Minor Changes diff --git a/packages/runtime/package.json b/packages/runtime/package.json index a8d420d60..c44f4a45b 100644 --- a/packages/runtime/package.json +++ b/packages/runtime/package.json @@ -1,6 +1,6 @@ { "name": "@openfn/runtime", - "version": "1.4.0", + "version": "1.4.1", "description": "Job processing runtime.", "type": "module", "exports": { diff --git a/packages/runtime/src/index.ts b/packages/runtime/src/index.ts index e47e08474..a773936d3 100644 --- a/packages/runtime/src/index.ts +++ b/packages/runtime/src/index.ts @@ -10,3 +10,5 @@ export * from './events'; export * from './errors'; export * from './modules/repo'; + +export * from './runtime-helpers'; diff --git a/packages/runtime/src/modules/linker.ts b/packages/runtime/src/modules/linker.ts index 42d848888..18a9cce2f 100644 --- a/packages/runtime/src/modules/linker.ts +++ b/packages/runtime/src/modules/linker.ts @@ -10,6 +10,11 @@ import { ImportError } from '../errors'; const defaultLogger = createMockLogger(); +// These specifiers are allowed to be imported "globally" +const moduleWhitelist: Record = { + '@openfn/runtime': true, +}; + export type ModuleInfo = { path?: string; version?: string; @@ -92,6 +97,13 @@ const linker: Linker = async (specifier, context, options = {}) => { // Loads a module as a general specifier or from a specific path const loadActualModule = async (specifier: string, options: LinkerOptions) => { const log = options.log || defaultLogger; + + // For a small number of whitelisted modules, import directly using basic module resolution + if (moduleWhitelist[specifier]) { + log.debug(`[linker] Importing whitelisted module: ${specifier}`); + return import(specifier); + } + const prefix = process.platform == 'win32' ? 'file://' : ''; // If the specifier is a path, just import it @@ -127,7 +139,6 @@ const loadActualModule = async (specifier: string, options: LinkerOptions) => { version = entry.version; } else { log.debug(`module not found in repo: ${specifier}`); - // throw new ImportError(`${specifier} not found in repo`); } } @@ -153,7 +164,6 @@ const loadActualModule = async (specifier: string, options: LinkerOptions) => { } } - // Generic error (we should never get here) throw new ImportError(`Failed to import module "${specifier}"`); }; diff --git a/packages/runtime/src/runtime-helpers.ts b/packages/runtime/src/runtime-helpers.ts new file mode 100644 index 000000000..092c13c89 --- /dev/null +++ b/packages/runtime/src/runtime-helpers.ts @@ -0,0 +1,26 @@ +/** + * Helper functions designed to be used in job code + */ + +import { State } from '@openfn/lexicon'; + +// Defer will take an operation with a promise chain +// and break it up into a deferred function call which +// ensures the operation is a promise +// eg, fn().then(s => s) + +export function defer( + fn: (s: State) => State, + complete = (p: Promise) => p, + error = (e: any, _state: State): void => { + throw e; + } +) { + return (state: State) => { + try { + return complete(Promise.resolve(fn(state)).catch((e) => error(e, state))); + } catch (e) { + return error(e, state); + } + }; +} diff --git a/packages/runtime/test/execute/expression.test.ts b/packages/runtime/test/execute/expression.test.ts index c60cf5d66..01dc93063 100644 --- a/packages/runtime/test/execute/expression.test.ts +++ b/packages/runtime/test/execute/expression.test.ts @@ -13,7 +13,7 @@ type TestState = State & { }; const createState = (data = {}) => ({ - data: data, + data, configuration: {}, }); @@ -335,6 +335,61 @@ test.serial('calls execute if exported from a job', async (t) => { t.is(logger._history.length, 1); }); +test.serial('handles a promise returned by an operation', async (t) => { + const logger = createMockLogger(undefined, { level: 'info' }); + + const job = `export default [ + (s) => new Promise((r) => r(s)) + ];`; + + const state = createState({ x: 1 }); + const context = createContext({ opts: { jobLogger: logger } }); + + const result = (await execute(context, job, state)) as TestState; + + t.is(result.data.x, 1); +}); + +test.serial( + 'handles a promise returned by an operation with .then()', + async (t) => { + const logger = createMockLogger(undefined, { level: 'info' }); + + const job = `export default [ + (s) => + new Promise((r) => r(s)) + .then(s => ({ data: { x: 2 }})) + ];`; + + const state = createState({ x: 1 }); + const context = createContext({ opts: { jobLogger: logger } }); + + const result = (await execute(context, job, state)) as TestState; + + t.is(result.data.x, 2); + } +); + +test.serial( + 'handles a promise returned by an operation with .catch()', + async (t) => { + const logger = createMockLogger(undefined, { level: 'info' }); + + const job = `export default [ + (s) => + new Promise((r) => { throw "err" }) + .catch((e) => ({ data: { x: 3 }})) + ];`; + + const state = createState({ x: 1 }); + const context = createContext({ opts: { jobLogger: logger } }); + + const result = (await execute(context, job, state)) as TestState; + + t.is(result.data.x, 3); + } +); + // Skipping for now as the default timeout is quite long test.skip('Throws after default timeout', async (t) => { const logger = createMockLogger(undefined, { level: 'info' }); diff --git a/packages/runtime/test/runtime-helpers.test.ts b/packages/runtime/test/runtime-helpers.test.ts new file mode 100644 index 000000000..c40b467a3 --- /dev/null +++ b/packages/runtime/test/runtime-helpers.test.ts @@ -0,0 +1,97 @@ +import test from 'ava'; +import { defer } from '../src/runtime-helpers'; + +test('defer does not execute immediately', (t) => { + let x = 0; + + const op = () => x++; + + defer(op); + + t.is(x, 0); +}); + +test('defer: function executes when called', async (t) => { + let x = 0; + + const op = () => x++; + + const fn = defer(op); + + await fn({}); + + t.is(x, 1); +}); + +test('defer: function executes an async function when called', async (t) => { + const op = () => + new Promise((resolve) => { + setTimeout(() => { + resolve(22); + }, 2); + }); + + const fn = defer(op); + + const result = await fn({}); + + t.is(result, 22); +}); + +test('defer: returns a value', async (t) => { + const op = (s) => s * s; + + const fn = defer(op); + + const result = await fn(5); + + t.is(result, 25); +}); + +test('defer: invoke the complete callback and pass state', async (t) => { + const op = (s) => ++s; + + const fn = defer(op, (p) => p.then((s) => (s *= 2))); + + const result = await fn(2); + + t.is(result, 6); +}); + +test('defer: catch an error', async (t) => { + const op = () => { + throw 'lamine yamal'; + }; + + const c = (e: any, s: any) => { + t.truthy(e); + t.truthy(s); + }; + + const fn = defer(op, undefined, c); + + await fn(1); +}); + +test('defer: catch an async error', async (t) => { + const op = () => + new Promise((_resolve, reject) => { + setTimeout(() => { + // This should be handled gracefully + reject('lamine yamal'); + + // but this will be uncaught! + // I don't think there's anything we can do about this tbh + //throw 'lamine yamal'; + }, 2); + }); + + const c = (e: any, s: any) => { + t.is(e, 'lamine yamal'); + t.truthy(s); + }; + + const fn = defer(op, undefined, c); + + await fn(1); +}); diff --git a/packages/ws-worker/CHANGELOG.md b/packages/ws-worker/CHANGELOG.md index 1d359c436..b660de64a 100644 --- a/packages/ws-worker/CHANGELOG.md +++ b/packages/ws-worker/CHANGELOG.md @@ -1,5 +1,18 @@ # ws-worker +## 1.4.0 + +### Minor Changes + +- Allow operations to behave like promises (ie, support fn().then()) + +### Patch Changes + +- Updated dependencies [40fd45b] +- Updated dependencies + - @openfn/runtime@1.4.1 + - @openfn/engine-multi@1.2.0 + ## 1.3.0 ### Minor Changes diff --git a/packages/ws-worker/package.json b/packages/ws-worker/package.json index 585355082..33135a3b7 100644 --- a/packages/ws-worker/package.json +++ b/packages/ws-worker/package.json @@ -1,6 +1,6 @@ { "name": "@openfn/ws-worker", - "version": "1.3.0", + "version": "1.4.0", "description": "A Websocket Worker to connect Lightning to a Runtime Engine", "main": "dist/index.js", "type": "module", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d02d70445..af598fe53 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -107,6 +107,46 @@ importers: specifier: ^3.0.2 version: 3.0.2 + integration-tests/execute: + dependencies: + '@openfn/compiler': + specifier: workspace:^ + version: link:../../packages/compiler + '@openfn/language-common': + specifier: 1.7.7 + version: 1.7.7 + '@openfn/language-http': + specifier: 6.4.0 + version: 6.4.0 + '@openfn/runtime': + specifier: workspace:^ + version: link:../../packages/runtime + '@types/node': + specifier: ^18.15.13 + version: 18.15.13 + ava: + specifier: 5.3.1 + version: 5.3.1 + date-fns: + specifier: ^2.30.0 + version: 2.30.0 + rimraf: + specifier: ^3.0.2 + version: 3.0.2 + ts-node: + specifier: 10.8.1 + version: 10.8.1(@types/node@18.15.13)(typescript@5.1.6) + tslib: + specifier: ^2.4.0 + version: 2.4.0 + typescript: + specifier: ^5.1.6 + version: 5.1.6 + devDependencies: + '@types/rimraf': + specifier: ^3.0.2 + version: 3.0.2 + integration-tests/worker: dependencies: '@openfn/engine-multi': @@ -1363,6 +1403,11 @@ packages: heap: 0.2.7 dev: false + /@fastify/busboy@2.1.1: + resolution: {integrity: sha512-vBZP4NlzfOlerQTnba4aqZoMhE/a9HY7HRqoOPaETQcSQuWEIyZMHGfVu6w9wGtGK5fED5qRs2DteVCjOH60sA==} + engines: {node: '>=14'} + dev: false + /@inquirer/checkbox@1.3.5: resolution: {integrity: sha512-ZznkPU+8XgNICKkqaoYENa0vTw9jeToEHYyG5gUKpGmY+4PqPTsvLpSisOt9sukLkYzPRkpSCHREgJLqbCG3Fw==} engines: {node: '>=14.18.0'} @@ -1619,6 +1664,22 @@ packages: semver: 7.5.4 dev: true + /@openfn/language-common@1.15.0: + resolution: {integrity: sha512-aBWCvnJc0MCRjF6wUHicU5nkM3wWxrJV7K81j0FB7hQqerFSTk/ceq8/a98bi2Tcd9CV8WBTPF1AfROMcpNSEg==} + dependencies: + ajv: 8.17.1 + axios: 1.1.3 + csv-parse: 5.5.6 + csvtojson: 2.0.10 + date-fns: 2.30.0 + http-status-codes: 2.3.0 + jsonpath-plus: 4.0.0 + lodash: 4.17.21 + undici: 5.28.4 + transitivePeerDependencies: + - debug + dev: false + /@openfn/language-common@1.7.5: resolution: {integrity: sha512-QivV3v5Oq5fb4QMopzyqUUh+UGHaFXBdsGr6RCmu6bFnGXdJdcQ7GpGpW5hKNq29CkmE23L/qAna1OLr4rP/0w==} dependencies: @@ -1630,10 +1691,31 @@ packages: - debug dev: true + /@openfn/language-common@1.7.7: + resolution: {integrity: sha512-GSoAbo6oL0b8jHufhLKvIzHJ271aE2AKv/ibeuiWU3CqN1gRmaHArlA/omlCs/rsfcieSp2VWAvWeGuFY8buZw==} + dependencies: + axios: 1.1.3 + date-fns: 2.30.0 + jsonpath-plus: 4.0.0 + lodash: 4.17.21 + transitivePeerDependencies: + - debug + dev: false + /@openfn/language-common@2.0.0-rc3: resolution: {integrity: sha512-7kwhBnCd1idyTB3MD9dXmUqROAhoaUIkz2AGDKuv9vn/cbZh7egEv9/PzKkRcDJYFV9qyyS+cVT3Xbgsg2ii5g==} bundledDependencies: [] + /@openfn/language-http@6.4.0: + resolution: {integrity: sha512-dZwbBV47UrmUlDo5Z9F5XMQq0i8XEHNo0xbgUcCeq7EaJrYn4E2EzK4q2DLzSZb+14K/PWOeUHATL/LCHx+w6g==} + dependencies: + '@openfn/language-common': 1.15.0 + cheerio: 1.0.0-rc.12 + cheerio-tableparser: 1.0.1 + transitivePeerDependencies: + - debug + dev: false + /@pkgjs/parseargs@0.11.0: resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==} engines: {node: '>=14'} @@ -1660,7 +1742,7 @@ packages: '@slack/logger': 3.0.0 '@slack/types': 2.8.0 '@types/is-stream': 1.1.0 - '@types/node': 18.15.3 + '@types/node': 18.15.13 axios: 0.27.2 eventemitter3: 3.1.2 form-data: 2.5.1 @@ -1753,7 +1835,7 @@ packages: /@types/gunzip-maybe@1.4.0: resolution: {integrity: sha512-dFP9GrYAR9KhsjTkWJ8q8Gsfql75YIKcg9DuQOj/IrlPzR7W+1zX+cclw1McV82UXAQ+Lpufvgk3e9bC8+HzgA==} dependencies: - '@types/node': 18.15.3 + '@types/node': 18.15.13 dev: true /@types/http-assert@1.5.3: @@ -1880,10 +1962,6 @@ packages: /@types/node@18.15.13: resolution: {integrity: sha512-N+0kuo9KgrUQ1Sn/ifDXsvg0TTleP7rIy4zOBGECxAljqvqfqpTfzx0Q1NUedOixRMBfe2Whhb056a42cWs26Q==} - /@types/node@18.15.3: - resolution: {integrity: sha512-p6ua9zBxz5otCmbpb5D3U4B5Nanw6Pk3PPyX05xnxbB/fRv71N7CPmORg7uAD5P70T0xmx1pzAx/FUfa5X+3cw==} - dev: true - /@types/node@20.4.5: resolution: {integrity: sha512-rt40Nk13II9JwQBdeYqmbn2Q6IVTA5uPhvSO+JVqdXw/6/4glI6oR9ezty/A9Hg5u7JH4OmYmuQ+XvjKm0Datg==} @@ -1960,7 +2038,7 @@ packages: /@types/tar-stream@2.2.2: resolution: {integrity: sha512-1AX+Yt3icFuU6kxwmPakaiGrJUwG44MpuiqPg4dSolRFk6jmvs4b3IbUol9wKDLIgU76gevn3EwE8y/DkSJCZQ==} dependencies: - '@types/node': 18.15.3 + '@types/node': 18.15.13 dev: true /@types/treeify@1.0.0: @@ -2088,6 +2166,15 @@ packages: clean-stack: 4.2.0 indent-string: 5.0.0 + /ajv@8.17.1: + resolution: {integrity: sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==} + dependencies: + fast-deep-equal: 3.1.3 + fast-uri: 3.0.1 + json-schema-traverse: 1.0.0 + require-from-string: 2.0.2 + dev: false + /ansi-colors@4.1.3: resolution: {integrity: sha512-/6w/C21Pm1A7aZitlI5Ni/2J6FFQN8i1Cvz3kHABAAbw93v/NlvKdVOqz7CCWz/3iv/JplRSEEZ83XION15ovw==} engines: {node: '>=6'} @@ -2248,7 +2335,6 @@ packages: /asynckit@0.4.0: resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==} - dev: true /atob@2.1.2: resolution: {integrity: sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg==} @@ -2396,7 +2482,6 @@ packages: proxy-from-env: 1.1.0 transitivePeerDependencies: - debug - dev: true /b4a@1.6.1: resolution: {integrity: sha512-AsKjNhz72yxteo/0EtQEiwkMUgk/tGmycXlbG4g3Ard2/ULtNLUykGOkeK0egmN27h0xMAhb76jYccW+XTBExA==} @@ -2477,9 +2562,17 @@ packages: readable-stream: 4.2.0 dev: true + /bluebird@3.7.2: + resolution: {integrity: sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==} + dev: false + /blueimp-md5@2.19.0: resolution: {integrity: sha512-DRQrD6gJyy8FbiE4s+bDoXS9hiW3Vbx5uCdwvcCf3zLHL+Iv7LtGHLpr+GZV8rHG8tK766FGYBwRbu8pELTt+w==} + /boolbase@1.0.0: + resolution: {integrity: sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==} + dev: false + /brace-expansion@1.1.11: resolution: {integrity: sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==} dependencies: @@ -2697,6 +2790,34 @@ packages: /chardet@0.7.0: resolution: {integrity: sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA==} + /cheerio-select@2.1.0: + resolution: {integrity: sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g==} + dependencies: + boolbase: 1.0.0 + css-select: 5.1.0 + css-what: 6.1.0 + domelementtype: 2.3.0 + domhandler: 5.0.3 + domutils: 3.1.0 + dev: false + + /cheerio-tableparser@1.0.1: + resolution: {integrity: sha512-SCSWdMoFvIue0jdFZqRNPXDCZ67vuirJEG3pfh3AAU2hwxe/qh1EQUkUNPWlZhd6DMjRlTfcpcPWbaowjwRnNQ==} + dev: false + + /cheerio@1.0.0-rc.12: + resolution: {integrity: sha512-VqR8m68vM46BNnuZ5NtnGBKIE/DfN0cRIzg9n40EIq9NOv90ayxLBXA8fXC5gquFRGJSTRqBq25Jt2ECLR431Q==} + engines: {node: '>= 6'} + dependencies: + cheerio-select: 2.1.0 + dom-serializer: 2.0.0 + domhandler: 5.0.3 + domutils: 3.1.0 + htmlparser2: 8.0.2 + parse5: 7.1.2 + parse5-htmlparser2-tree-adapter: 7.0.0 + dev: false + /chokidar@2.1.8: resolution: {integrity: sha512-ZmZUazfOzf0Nve7duiCKD23PFSCs4JPoYyccjUFF3aQkQadqBhfzhjkwBH2mNOG9cTBwhamM37EIsIkZw3nRgg==} deprecated: Chokidar 2 does not receive security updates since 2019. Upgrade to chokidar 3 with 15x fewer dependencies @@ -2869,7 +2990,6 @@ packages: engines: {node: '>= 0.8'} dependencies: delayed-stream: 1.0.0 - dev: true /commander@4.1.1: resolution: {integrity: sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA==} @@ -2994,6 +3114,21 @@ packages: which: 2.0.2 dev: true + /css-select@5.1.0: + resolution: {integrity: sha512-nwoRF1rvRRnnCqqY7updORDsuqKzqYJ28+oSMaJMMgOauh3fvwHqMS7EZpIPqK8GL+g9mKxF1vP/ZjSeNjEVHg==} + dependencies: + boolbase: 1.0.0 + css-what: 6.1.0 + domhandler: 5.0.3 + domutils: 3.1.0 + nth-check: 2.1.1 + dev: false + + /css-what@6.1.0: + resolution: {integrity: sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw==} + engines: {node: '>= 6'} + dev: false + /cssesc@3.0.0: resolution: {integrity: sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg==} engines: {node: '>=4'} @@ -3012,6 +3147,10 @@ packages: resolution: {integrity: sha512-cO1I/zmz4w2dcKHVvpCr7JVRu8/FymG5OEpmvsZYlccYolPBLoVGKUHgNoc4ZGkFeFlWGEDmMyBM+TTqRdW/wg==} dev: true + /csv-parse@5.5.6: + resolution: {integrity: sha512-uNpm30m/AGSkLxxy7d9yRXpJQFrZzVWLFBkS+6ngPcZkw/5k3L/jjFuj7tVnEpRn+QgmiXr21nDlhCiUK4ij2A==} + dev: false + /csv-stringify@5.6.5: resolution: {integrity: sha512-PjiQ659aQ+fUTQqSrd1XEDnOr52jh30RBurfzkscaE2tPaFsDH5wOAHJiw8XAHphRknCwMUE9KRayc4K/NbO8A==} dev: true @@ -3026,6 +3165,16 @@ packages: stream-transform: 2.1.3 dev: true + /csvtojson@2.0.10: + resolution: {integrity: sha512-lUWFxGKyhraKCW8Qghz6Z0f2l/PqB1W3AO0HKJzGIQ5JRSlR651ekJDiGJbBT4sRNNv5ddnSGVEnsxP9XRCVpQ==} + engines: {node: '>=4.0.0'} + hasBin: true + dependencies: + bluebird: 3.7.2 + lodash: 4.17.21 + strip-bom: 2.0.0 + dev: false + /currently-unhandled@0.4.1: resolution: {integrity: sha512-/fITjgjGU50vjQ4FH6eUoYu+iUoUKIXws2hL15JJpIR+BbTxaXQsMuuyjtNh2WqsSBS5nsaZHFsFecyw5CCAng==} engines: {node: '>=0.10.0'} @@ -3166,7 +3315,6 @@ packages: /delayed-stream@1.0.0: resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} engines: {node: '>=0.4.0'} - dev: true /delegates@1.0.0: resolution: {integrity: sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==} @@ -3217,6 +3365,33 @@ packages: resolution: {integrity: sha512-+HlytyjlPKnIG8XuRG8WvmBP8xs8P71y+SKKS6ZXWoEgLuePxtDoUEiH7WkdePWrQ5JBpE6aoVqfZfJUQkjXwA==} dev: true + /dom-serializer@2.0.0: + resolution: {integrity: sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==} + dependencies: + domelementtype: 2.3.0 + domhandler: 5.0.3 + entities: 4.5.0 + dev: false + + /domelementtype@2.3.0: + resolution: {integrity: sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==} + dev: false + + /domhandler@5.0.3: + resolution: {integrity: sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==} + engines: {node: '>= 4'} + dependencies: + domelementtype: 2.3.0 + dev: false + + /domutils@3.1.0: + resolution: {integrity: sha512-H78uMmQtI2AhgDJjWeQmHwJJ2bLPD3GMmO7Zja/ZZh84wkm+4ut+IUnUdRa8uCGX88DiVx1j6FRe1XfxEgjEZA==} + dependencies: + dom-serializer: 2.0.0 + domelementtype: 2.3.0 + domhandler: 5.0.3 + dev: false + /dreamopt@0.8.0: resolution: {integrity: sha512-vyJTp8+mC+G+5dfgsY+r3ckxlz+QMX40VjPQsZc5gxVAxLmi64TBoVkP54A/pRAXMXsbu2GMMBrZPxNv23waMg==} engines: {node: '>=0.4.0'} @@ -3278,6 +3453,11 @@ packages: ansi-colors: 4.1.3 dev: true + /entities@4.5.0: + resolution: {integrity: sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==} + engines: {node: '>=0.12'} + dev: false + /err-code@2.0.3: resolution: {integrity: sha512-2bmlRpNKBxT/CRmPOlyISQpNj+qSeYvcym/uT0Jx2bMOlKLtSy1ZmLuVxSEKKyor/N5yhvp/ZiG1oE3DEYMSFA==} dev: true @@ -3992,6 +4172,10 @@ packages: - supports-color dev: true + /fast-deep-equal@3.1.3: + resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} + dev: false + /fast-diff@1.3.0: resolution: {integrity: sha512-VxPP4NqbUjj6MaAOafWeUn2cXWLcCtljklUtZf0Ind4XQ+QPtmA0b18zZy0jIQx+ExRVCR/ZQpBmik5lXshNsw==} @@ -4032,6 +4216,10 @@ packages: resolution: {integrity: sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA==} dev: false + /fast-uri@3.0.1: + resolution: {integrity: sha512-MWipKbbYiYI0UC7cl8m/i/IWTqfC8YXsqjzybjddLsFjStroQzsHXkc73JutMvBiXmOvapk+axIl79ig5t55Bw==} + dev: false + /fastq@1.13.0: resolution: {integrity: sha512-YpkpUnK8od0o1hmeSc7UUs/eB/vIPWJYjKck2QKIzAf71Vm1AAQ3EbuZB3g2JIy+pg+ERD0vqI79KyZiB2e2Nw==} dependencies: @@ -4138,7 +4326,6 @@ packages: peerDependenciesMeta: debug: optional: true - dev: true /for-in@1.0.2: resolution: {integrity: sha512-7EwmXrOjyL+ChxMhmG5lnW9MPt1aIeZEwKhQzoBUdTV0N3zuwWDZYVJatDvZ2OyzPUvdIAZDsCetk3coyMfcnQ==} @@ -4169,7 +4356,6 @@ packages: asynckit: 0.4.0 combined-stream: 1.0.8 mime-types: 2.1.35 - dev: true /fragment-cache@0.2.1: resolution: {integrity: sha512-GMBAbW9antB8iZRHLoGw0b3HANt57diZYFO/HL1JGIC1MjKrdmhxvrJbupnVvpys0zsz7yBApXdQyfepKly2kA==} @@ -4477,6 +4663,15 @@ packages: lru-cache: 7.18.3 dev: true + /htmlparser2@8.0.2: + resolution: {integrity: sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA==} + dependencies: + domelementtype: 2.3.0 + domhandler: 5.0.3 + domutils: 3.1.0 + entities: 4.5.0 + dev: false + /http-assert@1.5.0: resolution: {integrity: sha512-uPpH7OKX4H25hBmU6G1jWNaqJGpTXxey+YOUizJUAgu0AjLUeC8D73hTrhvDS5D+GJN1DN1+hhc/eF/wpxtp0w==} engines: {node: '>= 0.8'} @@ -4545,6 +4740,10 @@ packages: - supports-color dev: true + /http-status-codes@2.3.0: + resolution: {integrity: sha512-RJ8XvFvpPM/Dmc5SV+dC4y5PCeOhT3x1Hq0NU3rjGeg5a/CqlhZ7uudknPwZFz4aeAXDcbAyaeP7GAo9lvngtA==} + dev: false + /https-proxy-agent@5.0.1: resolution: {integrity: sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==} engines: {node: '>= 6'} @@ -4945,6 +5144,10 @@ packages: resolution: {integrity: sha512-43r2mRvz+8JRIKnWJ+3j8JtjRKZ6GmjzfaE/qiBJnikNnYv/6bagRJ1kUhNk8R5EX/GkobD+r+sfxCPJsiKBLQ==} engines: {node: '>=12'} + /is-utf8@0.2.1: + resolution: {integrity: sha512-rMYPYvCzsXywIsldgLaSoPlw5PfoB/ssr7hY4pLfcodrA5M/eArza1a9VmTiNIBNMjOGr1Ow9mTyU2o69U6U9Q==} + dev: false + /is-weakref@1.0.2: resolution: {integrity: sha512-qctsuLZmIQ0+vSSMfoVvyFe2+GSEvnmZ2ezTup1SBse9+twCCeial6EEi3Nc2KFcf6+qz2FBPnjXsk8xhKSaPQ==} dependencies: @@ -5038,6 +5241,10 @@ packages: resolution: {integrity: sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==} dev: true + /json-schema-traverse@1.0.0: + resolution: {integrity: sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==} + dev: false + /jsonfile@4.0.0: resolution: {integrity: sha512-m6F1R3z8jjlf2imQHS2Qez5sjKWQzbuuhuJ/FKYFRZvPE3PuHcSMVZzfsLhGVOkfd20obL5SWEBew5ShlquNxg==} optionalDependencies: @@ -5052,7 +5259,6 @@ packages: /jsonpath-plus@4.0.0: resolution: {integrity: sha512-e0Jtg4KAzDJKKwzbLaUtinCn0RZseWBVRTRGihSpvFlM3wTR7ExSp+PTdeTsDrLNJUe7L7JYJe8mblHX5SCT6A==} engines: {node: '>=10.0'} - dev: true /jsonpath@1.1.1: resolution: {integrity: sha512-l6Cg7jRpixfbgoWgkrl77dgEj8RPvND0wMH6TwQmi9Qs4TFfS9u5cUFnbeKTwj5ga5Y3BTGGNI28k117LJ009w==} @@ -5784,6 +5990,12 @@ packages: path-key: 3.1.1 dev: true + /nth-check@2.1.1: + resolution: {integrity: sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==} + dependencies: + boolbase: 1.0.0 + dev: false + /object-assign@4.1.1: resolution: {integrity: sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==} engines: {node: '>=0.10.0'} @@ -6053,6 +6265,19 @@ packages: resolution: {integrity: sha512-Tpb8Z7r7XbbtBTrM9UhpkzzaMrqA2VXMT3YChzYltwV3P3pM6t8wl7TvpMnSTosz1aQAdVib7kdoys7vYOPerw==} engines: {node: '>=12'} + /parse5-htmlparser2-tree-adapter@7.0.0: + resolution: {integrity: sha512-B77tOZrqqfUfnVcOrUvfdLbz4pu4RopLD/4vmu3HUPswwTA8OH0EMW9BlWR2B0RCoiZRAHEUu7IxeP1Pd1UU+g==} + dependencies: + domhandler: 5.0.3 + parse5: 7.1.2 + dev: false + + /parse5@7.1.2: + resolution: {integrity: sha512-Czj1WaSVpaoj0wbhMzLmWD69anp2WH7FXMB9n1Sy8/ZFF9jolSQVMu1Ij5WIyGmcBmhk7EOndpO4mIpihVqAXw==} + dependencies: + entities: 4.5.0 + dev: false + /parseurl@1.3.3: resolution: {integrity: sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==} engines: {node: '>= 0.8'} @@ -6380,7 +6605,6 @@ packages: /proxy-from-env@1.1.0: resolution: {integrity: sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==} - dev: true /proxy-middleware@0.15.0: resolution: {integrity: sha512-EGCG8SeoIRVMhsqHQUdDigB2i7qU7fCsWASwn54+nPutYO8n4q6EiwMzyfWlC+dzRFExP+kvcnDFdBDHoZBU7Q==} @@ -6619,6 +6843,11 @@ packages: resolution: {integrity: sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==} engines: {node: '>=0.10.0'} + /require-from-string@2.0.2: + resolution: {integrity: sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==} + engines: {node: '>=0.10.0'} + dev: false + /require-main-filename@2.0.0: resolution: {integrity: sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==} dev: true @@ -7142,6 +7371,13 @@ packages: dependencies: ansi-regex: 6.0.1 + /strip-bom@2.0.0: + resolution: {integrity: sha512-kwrX1y7czp1E69n2ajbG65mIo9dqvJ+8aBQXOGVxqwvNbsXdFM6Lq37dLAY3mknUwru8CfcCbfOLL/gMo+fi3g==} + engines: {node: '>=0.10.0'} + dependencies: + is-utf8: 0.2.1 + dev: false + /strip-bom@3.0.0: resolution: {integrity: sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA==} engines: {node: '>=4'} @@ -7762,6 +7998,13 @@ packages: resolution: {integrity: sha512-hEQt0+ZLDVUMhebKxL4x1BTtDY7bavVofhZ9KZ4aI26X9SRaE+Y3m83XUL1UP2jn8ynjndwCCpEHdUG+9pP1Tw==} dev: true + /undici@5.28.4: + resolution: {integrity: sha512-72RFADWFqKmUb2hmmvNODKL3p9hcB6Gt2DOQMis1SEBaV6a4MH8soBvzg+95CYhCKPFedut2JY9bMfrDl9D23g==} + engines: {node: '>=14.0'} + dependencies: + '@fastify/busboy': 2.1.1 + dev: false + /union-value@1.0.1: resolution: {integrity: sha512-tJfXmxMeWYnczCVs7XAEvIV7ieppALdyepWMkHkwciRpZraG/xwT+s2JN8+pr1+8jCRf80FFzvr+MpQeeoF4Xg==} engines: {node: '>=0.10.0'}