Merge pull request #722 from OpenFn/promises-all-the-way-down

Support promises

integration-tests/execute/CHANGELOG.md

@@ -0,0 +1,10 @@
+# @openfn/integration-tests-execute
+
+## 1.0.1
+
+### Patch Changes
+
+- Updated dependencies [40fd45b]
+- Updated dependencies [40fd45b]
+  - @openfn/compiler@0.2.0
+  - @openfn/runtime@1.4.1

integration-tests/execute/package.json

@@ -0,0 +1,32 @@
+{
+  "name": "@openfn/integration-tests-execute",
+  "private": true,
+  "version": "1.0.1",
+  "description": "Job execution tests",
+  "author": "Open Function Group <[email protected]>",
+  "license": "ISC",
+  "type": "module",
+  "scripts": {
+    "test": "pnpm ava"
+  },
+  "dependencies": {
+    "@openfn/compiler": "workspace:^",
+    "@openfn/language-common": "1.7.7",
+    "@openfn/language-http": "6.4.0",
+    "@openfn/runtime": "workspace:^",
+    "@types/node": "^18.15.13",
+    "ava": "5.3.1",
+    "date-fns": "^2.30.0",
+    "rimraf": "^3.0.2",
+    "ts-node": "10.8.1",
+    "tslib": "^2.4.0",
+    "typescript": "^5.1.6"
+  },
+  "files": [
+    "dist",
+    "README.md"
+  ],
+  "devDependencies": {
+    "@types/rimraf": "^3.0.2"
+  }
+}

integration-tests/execute/readme.md

@@ -0,0 +1,5 @@
+This is a suite of examples of jobs.
+
+We don't really have a place where we can just write and test arbtirary job code with compilation.
+
+You can do it through the CLI or worker but they have significant overheads.

integration-tests/execute/src/execute.ts

@@ -0,0 +1,35 @@
+import path from 'node:path';
+import run from '@openfn/runtime';
+import compiler from '@openfn/compiler';
+
+const execute = async (job: string, state: any, adaptor = 'common') => {
+  // compile with common and dumb imports
+  const options = {
+    'add-imports': {
+      adaptor: {
+        name: `@openfn/language-${adaptor}`,
+        exportAll: true,
+      },
+    },
+  };
+  const compiled = compiler(job, options);
+  // console.log(compiled);
+
+  const result = await run(compiled, state, {
+    // preload the linker with some locally installed modules
+    linker: {
+      modules: {
+        '@openfn/language-common': {
+          path: path.resolve('node_modules/@openfn/language-common'),
+        },
+        '@openfn/language-http': {
+          path: path.resolve('node_modules/@openfn/language-http'),
+        },
+      },
+    },
+  });
+
+  return result;
+};
+
+export default execute;

integration-tests/execute/src/index.ts

@@ -0,0 +1,3 @@
+import execute from './execute';
+
+export default execute;

integration-tests/execute/test/execute.test.ts

@@ -0,0 +1,148 @@
+import test from 'ava';
+
+import execute from '../src/execute';
+
+const wait = `function wait() {
+  return (state) =>
+    new Promise((resolve) => {
+      setTimeout(() => resolve(state), 2);
+    });
+};`;
+
+test.serial('should return state', async (t) => {
+  const state = { data: { x: 1 } };
+
+  const job = `
+    fn(s => s)
+  `;
+  const result = await execute(job, state);
+
+  t.deepEqual(state, result);
+});
+
+test.serial('should use .then()', async (t) => {
+  const state = { data: { x: 1 } };
+
+  const job = `
+    fn(s => s)
+      .then((s) =>
+        ({
+          data: { x: 33 }
+        })
+      )
+  `;
+  const result = await execute(job, state);
+
+  t.deepEqual(result, { data: { x: 33 } });
+});
+
+test.serial('should chain .then() with state', async (t) => {
+  const state = { data: { x: 1 } };
+
+  const job = `
+    fn(s => ({ x: 1 }))
+      .then((s) =>
+        ({
+          x: s.x + 1
+        })
+      )
+  `;
+  const result = await execute(job, state);
+
+  t.deepEqual(result, { x: 2 });
+});
+
+test.serial('should use .then() as an argument', async (t) => {
+  const state = {};
+
+  const job = `fn(
+    fn(() => ({ x: 5 })).then((s) => ({ x: s.x + 1}))
+  )`;
+  const result = await execute(job, state);
+
+  t.deepEqual(result, { x: 6 });
+});
+
+test.serial('use then() with wait()', async (t) => {
+  const state = {
+    data: {
+      x: 22,
+    },
+  };
+
+  const job = `${wait}
+  wait().then(fn(s => s))`;
+
+  const result = await execute(job, state);
+
+  t.deepEqual(result.data, { x: 22 });
+});
+
+test.serial('catch an error and return it', async (t) => {
+  const state = {
+    data: {
+      x: 22,
+    },
+  };
+
+  const job = `fn(() => {
+    throw { err: true }
+  }).catch(e => e)`;
+
+  const result = await execute(job, state);
+  t.deepEqual(result, { err: true });
+});
+
+test.serial('catch an error and re-throw it', async (t) => {
+  const state = {
+    data: {
+      x: 22,
+    },
+  };
+
+  const job = `fn(() => {
+    throw new Error('err')
+  }).catch(e => { throw e })`;
+
+  const result = await execute(job, state);
+  t.is(result.errors['job-1'].name, 'JobError');
+  t.is(result.errors['job-1'].message, 'err');
+});
+
+test.serial('catch an error and return state', async (t) => {
+  const state = {
+    data: {
+      x: 22,
+    },
+  };
+
+  const job = `fn(() => {
+    throw { err: true }
+  }).catch((e, s) => s)`;
+
+  const result = await execute(job, state);
+  t.deepEqual(result, state);
+});
+
+test.serial('each with then ', async (t) => {
+  const state = {
+    ids: [1, 2, 3],
+    results: [],
+  };
+
+  const job = `each($.ids,
+      get(\`https://jsonplaceholder.typicode.com/todos/\${$.data}\`).then(
+      (s) => {
+        s.results.push(s.data);
+        return s;
+      }
+    )
+  )`;
+
+  const result = await execute(job, state, 'http');
+
+  t.is(result.results.length, 3);
+  t.is(result.results[0].id, 1);
+  t.is(result.results[1].id, 2);
+  t.is(result.results[2].id, 3);
+});

integration-tests/execute/tsconfig.json

@@ -0,0 +1,14 @@
+{
+  "ts-node": {
+    "experimentalSpecifierResolution": "node"
+  },
+  "compilerOptions": {
+    "allowSyntheticDefaultImports": true,
+    "module": "es2020",
+    "moduleResolution": "node",
+    "allowJs": true,
+    "isolatedModules": true,
+    "noEmit": true,
+    "skipLibCheck": true
+  }
+}

integration-tests/worker/CHANGELOG.md

@@ -1,5 +1,14 @@
 # @openfn/integration-tests-worker
 
+## 1.0.51
+
+### Patch Changes
+
+- Updated dependencies
+  - @openfn/engine-multi@1.2.0
+  - @openfn/ws-worker@1.4.0
+  - @openfn/lightning-mock@2.0.14
+
 ## 1.0.50
 
 ### Patch Changes

integration-tests/worker/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@openfn/integration-tests-worker",
   "private": true,
-  "version": "1.0.50",
+  "version": "1.0.51",
   "description": "Lightning WOrker integration tests",
   "author": "Open Function Group <[email protected]>",
   "license": "ISC",

packages/cli/CHANGELOG.md

@@ -1,5 +1,18 @@
 # @openfn/cli
 
+## 1.7.0
+
+### Minor Changes
+
+- Allow operations to behave like promises (ie, support fn().then())
+
+### Patch Changes
+
+- Updated dependencies [40fd45b]
+- Updated dependencies [40fd45b]
+  - @openfn/compiler@0.2.0
+  - @openfn/runtime@1.4.1
+
 ## 1.6.1
 
 ### Patch Changes

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openfn/cli",
-  "version": "1.6.1",
+  "version": "1.7.0",
   "description": "CLI devtools for the openfn toolchain.",
   "engines": {
     "node": ">=18",

packages/compiler/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @openfn/compiler
 
+## 0.2.0
+
+### Minor Changes
+
+- 40fd45b: Add promises transformer
+  Don't try and import variables declared in other import statements
+
 ## 0.1.4
 
 ### Patch Changes

packages/compiler/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openfn/compiler",
-  "version": "0.1.4",
+  "version": "0.2.0",
   "description": "Compiler and language tooling for openfn jobs.",
   "author": "Open Function Group <[email protected]>",
   "license": "ISC",

packages/compiler/src/transform.ts

@@ -6,6 +6,7 @@ import createLogger, { Logger } from '@openfn/logger';
 import addImports, { AddImportsOptions } from './transforms/add-imports';
 import ensureExports from './transforms/ensure-exports';
 import lazyState from './transforms/lazy-state';
+import promises from './transforms/promises';
 import topLevelOps, {
   TopLevelOpsOptions,
 } from './transforms/top-level-operations';
@@ -38,6 +39,7 @@ export type TransformOptions = {
   ['top-level-operations']?: TopLevelOpsOptions | boolean;
   ['test']?: any;
   ['lazy-state']?: any;
+  ['promises']?: any;
 };
 
 const defaultLogger = createLogger();
@@ -50,6 +52,7 @@ export default function transform(
   if (!transformers) {
     transformers = [
       lazyState,
+      promises,
       ensureExports,
       topLevelOps,
       addImports,

packages/compiler/src/transforms/add-imports.ts

@@ -108,6 +108,11 @@ export function findAllDanglingIdentifiers(ast: ASTNode) {
   const result: IdentifierList = {};
   visit(ast, {
     visitIdentifier: function (path) {
+      // If this is part of an import statement, do nothing
+      if (n.ImportSpecifier.check(path.parent.node)) {
+        return false;
+      }
+
       // undefined and NaN are treated as a regular identifier
       if (path.node.name === 'undefined' || path.node.name === 'NaN') {
         return false;