diff --git a/.gitignore b/.gitignore index f06235c..39b2c11 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,11 @@ node_modules dist + +# CloudQuery cache directory +.cq + +# Output directory for CloudQuery +data + +# CloudQuery log file +cloudquery.log diff --git a/README.md b/README.md index 4a9ebb2..b32a647 100644 --- a/README.md +++ b/README.md @@ -3,3 +3,7 @@ A [CloudQuery](https://www.cloudquery.io/) source plugin for [NS1](https://ns1.com/). Written in TypeScript using the recently added [CloudQuery JavaScript SDK](https://www.cloudquery.io/blog/announcing-cloudquery-javascript-sdk). + +## Developing +1. Start the plugin via `npm start` +2. Use the plugin via `npm run test:integration` diff --git a/config.yml b/config.yml new file mode 100644 index 0000000..b1095d1 --- /dev/null +++ b/config.yml @@ -0,0 +1,17 @@ +--- +kind: source +spec: + name: "ns1" + registry: "grpc" + path: "localhost:7777" + tables: ['*'] + destinations: ['file'] +--- +kind: destination +spec: + name: "file" + path: "cloudquery/file" + version: "v3.4.6" + spec: + path: "data/{{TABLE}}/{{UUID}}.{{FORMAT}}" + format: "csv" diff --git a/package.json b/package.json index 86d2bbd..7489dbb 100644 --- a/package.json +++ b/package.json @@ -8,7 +8,8 @@ "lint": "eslint src/** --ext .ts --no-error-on-unmatched-pattern", "format": "prettier --write src/**/*.ts", "build": "tsc", - "start": "ts-node --esm src/index.ts serve" + "start": "ts-node --esm src/index.ts serve", + "test:integration": "cloudquery sync config.yml --log-level debug" }, "dependencies": { "@cloudquery/plugin-sdk-javascript": "^0.0.6", diff --git a/src/tables.ts b/src/tables.ts index 53d788e..ade6a22 100644 --- a/src/tables.ts +++ b/src/tables.ts @@ -1,4 +1,12 @@ -import type { Table } from '@cloudquery/plugin-sdk-javascript/schema/table'; +import type { Writable } from 'stream'; +import { Utf8 } from '@cloudquery/plugin-sdk-javascript/arrow'; +import { createColumn } from '@cloudquery/plugin-sdk-javascript/schema/column'; +import { pathResolver } from '@cloudquery/plugin-sdk-javascript/schema/resolvers'; +import { createTable } from '@cloudquery/plugin-sdk-javascript/schema/table'; +import type { + Table, + TableResolver, +} from '@cloudquery/plugin-sdk-javascript/schema/table'; import type { Logger } from 'winston'; import type { Spec } from './spec.js'; @@ -6,6 +14,37 @@ export const getTables = async ( logger: Logger, spec: Spec, ): Promise => { - // TOD add tables - return Promise.resolve([]); + const resolver: TableResolver = async ( + clientMeta: unknown, + parent: unknown, + stream: Writable, + ) => { + await new Promise((_) => setTimeout(_, 500)); + + const data = new Array(10).fill(undefined); + data.forEach((_, index) => { + stream.write({ id: [index], name: [`hello ${index}`] }); + }); + return; + }; + + const table = createTable({ + name: 'test', + columns: [ + createColumn({ + name: 'id', + type: new Utf8(), + resolver: pathResolver('id'), + }), + createColumn({ + name: 'name', + type: new Utf8(), + resolver: pathResolver('name'), + }), + ], + description: 'testing', + resolver, + }); + + return Promise.resolve([table]); };