From 44448af34fa98345358b41621f5a2a51ec6b7f80 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Tue, 20 Dec 2022 15:41:20 +0100 Subject: [PATCH 1/3] use new manifest yaml --- .../StreamTestingPanel/ConfigMenu.tsx | 16 ++++++++++++-- .../src/components/connectorBuilder/types.ts | 22 +++++++++++++------ .../PatchedConnectorManifest.ts | 8 ------- .../ConnectorBuilderStateService.tsx | 5 +++-- .../connector_manifest_openapi.yaml | 2 +- 5 files changed, 33 insertions(+), 20 deletions(-) delete mode 100644 airbyte-webapp/src/core/domain/connectorBuilder/PatchedConnectorManifest.ts diff --git a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/ConfigMenu.tsx b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/ConfigMenu.tsx index 21f45bb9e26a..21ec2bc416f6 100644 --- a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/ConfigMenu.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/ConfigMenu.tsx @@ -9,6 +9,7 @@ import { InfoBox } from "components/ui/InfoBox"; import { Modal, ModalBody } from "components/ui/Modal"; import { Tooltip } from "components/ui/Tooltip"; +import { SourceDefinitionSpecificationDraft } from "core/domain/connector"; import { useConnectorBuilderState } from "services/connectorBuilder/ConnectorBuilderStateService"; import { ConnectorForm } from "views/Connector/ConnectorForm"; @@ -35,6 +36,17 @@ export const ConfigMenu: React.FC = ({ className }) => { setIsOpen(false); }; + const connectorDefinitionSpecification: SourceDefinitionSpecificationDraft | undefined = useMemo( + () => + jsonManifest.spec + ? { + documentationUrl: jsonManifest.spec.documentation_url, + connectionSpecification: jsonManifest.spec.connection_specification, + } + : undefined, + [jsonManifest] + ); + return ( <> = ({ className }) => { )} - {isOpen && jsonManifest.spec && ( + {isOpen && connectorDefinitionSpecification && ( setIsOpen(false)} @@ -87,7 +99,7 @@ export const ConfigMenu: React.FC = ({ className }) => { formType="source" bodyClassName={styles.formContent} footerClassName={styles.inputFormModalFooter} - selectedConnectorDefinitionSpecification={jsonManifest.spec} + selectedConnectorDefinitionSpecification={connectorDefinitionSpecification} formValues={formValues} onSubmit={async (values) => { setConfigString(JSON.stringify(values.connectionConfiguration, null, 2) ?? ""); diff --git a/airbyte-webapp/src/components/connectorBuilder/types.ts b/airbyte-webapp/src/components/connectorBuilder/types.ts index ff673ec7175d..809901e45224 100644 --- a/airbyte-webapp/src/components/connectorBuilder/types.ts +++ b/airbyte-webapp/src/components/connectorBuilder/types.ts @@ -1,6 +1,10 @@ import * as yup from "yup"; -import { ConnectorManifest, DeclarativeStream } from "core/request/ConnectorManifest"; +import { + ConnectorManifest, + DeclarativeStream, + InterpolatedRequestOptionsProvider, +} from "core/request/ConnectorManifest"; export interface BuilderFormValues { global: { @@ -45,36 +49,40 @@ export const builderFormValidationSchema = yup.object().shape({ export const convertToManifest = (values: BuilderFormValues): ConnectorManifest => { const manifestStreams: DeclarativeStream[] = values.streams.map((stream) => { return { + type: "DeclarativeStream", name: stream.name, retriever: { + type: "SimpleRetriever", name: stream.name, requester: { + type: "HttpRequester", name: stream.name, url_base: values.global?.urlBase, path: stream.urlPath, request_options_provider: { + // TODO can't declar type here because the server will error out, but the types dictate it is needed. Fix here once server is fixed. + // type: "InterpolatedRequestOptionsProvider", request_parameters: Object.fromEntries(stream.requestOptions.requestParameters), request_headers: Object.fromEntries(stream.requestOptions.requestHeaders), request_body_data: Object.fromEntries(stream.requestOptions.requestBody), - }, - // TODO: remove these empty "config" values once they are no longer required in the connector manifest JSON schema - config: {}, + } as InterpolatedRequestOptionsProvider, }, record_selector: { + type: "RecordSelector", extractor: { + type: "DpathExtractor", field_pointer: stream.fieldPointer, - config: {}, }, }, - config: {}, }, - config: {}, }; }); return { version: "0.1.0", + type: "DeclarativeSource", check: { + type: "CheckStream", stream_names: [], }, streams: manifestStreams, diff --git a/airbyte-webapp/src/core/domain/connectorBuilder/PatchedConnectorManifest.ts b/airbyte-webapp/src/core/domain/connectorBuilder/PatchedConnectorManifest.ts deleted file mode 100644 index 2e8dd79b575f..000000000000 --- a/airbyte-webapp/src/core/domain/connectorBuilder/PatchedConnectorManifest.ts +++ /dev/null @@ -1,8 +0,0 @@ -import { SourceDefinitionSpecificationDraft } from "core/domain/connector"; - -import { ConnectorManifest } from "../../request/ConnectorManifest"; - -// Patching this type as required until the upstream schema is updated -export interface PatchedConnectorManifest extends ConnectorManifest { - spec?: SourceDefinitionSpecificationDraft; -} diff --git a/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx b/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx index 0f4ec2c67548..78c38bd1dcb9 100644 --- a/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx +++ b/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx @@ -5,7 +5,6 @@ import { useLocalStorage } from "react-use"; import { BuilderFormValues, convertToManifest } from "components/connectorBuilder/types"; -import { PatchedConnectorManifest } from "core/domain/connectorBuilder/PatchedConnectorManifest"; import { StreamReadRequestBodyConfig, StreamsListReadStreamsItem } from "core/request/ConnectorBuilderClient"; import { ConnectorManifest } from "core/request/ConnectorManifest"; @@ -21,7 +20,9 @@ export const DEFAULT_BUILDER_FORM_VALUES: BuilderFormValues = { const DEFAULT_JSON_MANIFEST_VALUES: ConnectorManifest = { version: "0.1.0", + type: "DeclarativeSource", check: { + type: "CheckStream", stream_names: [], }, streams: [], @@ -32,7 +33,7 @@ export type BuilderView = "global" | number; interface Context { builderFormValues: BuilderFormValues; - jsonManifest: PatchedConnectorManifest; + jsonManifest: ConnectorManifest; yamlManifest: string; yamlEditorIsMounted: boolean; yamlIsValid: boolean; diff --git a/airbyte-webapp/src/services/connectorBuilder/connector_manifest_openapi.yaml b/airbyte-webapp/src/services/connectorBuilder/connector_manifest_openapi.yaml index 89fe28623293..690df8ae8fe4 100644 --- a/airbyte-webapp/src/services/connectorBuilder/connector_manifest_openapi.yaml +++ b/airbyte-webapp/src/services/connectorBuilder/connector_manifest_openapi.yaml @@ -6,4 +6,4 @@ paths: {} components: schemas: ConnectorManifest: - $ref: "../../../../airbyte-cdk/python/airbyte_cdk/sources/declarative/config_component_schema.json" + $ref: "../../../../airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_component_schema.yaml" From 72d7697887d2497a1b3444d1a93f076e1bd2f93e Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 21 Dec 2022 10:31:10 +0100 Subject: [PATCH 2/3] Update airbyte-webapp/src/components/connectorBuilder/types.ts Co-authored-by: Lake Mossman --- airbyte-webapp/src/components/connectorBuilder/types.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-webapp/src/components/connectorBuilder/types.ts b/airbyte-webapp/src/components/connectorBuilder/types.ts index 809901e45224..5f3a5cc2a7ac 100644 --- a/airbyte-webapp/src/components/connectorBuilder/types.ts +++ b/airbyte-webapp/src/components/connectorBuilder/types.ts @@ -60,7 +60,7 @@ export const convertToManifest = (values: BuilderFormValues): ConnectorManifest url_base: values.global?.urlBase, path: stream.urlPath, request_options_provider: { - // TODO can't declar type here because the server will error out, but the types dictate it is needed. Fix here once server is fixed. + // TODO can't declare type here because the server will error out, but the types dictate it is needed. Fix here once server is fixed. // type: "InterpolatedRequestOptionsProvider", request_parameters: Object.fromEntries(stream.requestOptions.requestParameters), request_headers: Object.fromEntries(stream.requestOptions.requestHeaders), From 27835061b1c67c2b9bb914acbb5f7f473ba88f1e Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Thu, 22 Dec 2022 14:05:09 +0100 Subject: [PATCH 3/3] use updated manifest types --- .../Builder/InjectRequestOptionFields.tsx | 2 +- .../Builder/PaginationSection.tsx | 10 +++++++-- .../Builder/StreamSlicerSection.tsx | 13 ++++++++---- .../Builder/ToggleGroupField.tsx | 11 +++++----- .../StreamTestingPanel/StreamTestingPanel.tsx | 9 +++----- .../src/components/connectorBuilder/types.ts | 21 +++++++++++-------- .../ConnectorBuilderStateService.tsx | 7 ++++--- 7 files changed, 43 insertions(+), 30 deletions(-) diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/InjectRequestOptionFields.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/InjectRequestOptionFields.tsx index 7f556b44cc0e..3d8607eb85f9 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/InjectRequestOptionFields.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/InjectRequestOptionFields.tsx @@ -30,7 +30,7 @@ export const InjectRequestOptionFields: React.FC } onChange={(newValue) => { if (newValue === "path") { - helpers.setValue({ inject_into: newValue, field_name: undefined }); + helpers.setValue({ inject_into: newValue, field_name: undefined, type: "RequestOption" }); } }} label="Inject into" diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/PaginationSection.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/PaginationSection.tsx index 4082e34cf04e..0a6878c28fa0 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/PaginationSection.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/PaginationSection.tsx @@ -3,6 +3,9 @@ import { useField } from "formik"; import GroupControls from "components/GroupControls"; import { ControlLabels } from "components/LabeledControl"; +import { RequestOption } from "core/request/ConnectorManifest"; + +import { BuilderPaginator } from "../types"; import { BuilderCard } from "./BuilderCard"; import { BuilderField } from "./BuilderField"; import { BuilderOneOf } from "./BuilderOneOf"; @@ -14,7 +17,7 @@ interface PaginationSectionProps { } export const PaginationSection: React.FC = ({ streamFieldPath }) => { - const [field, , helpers] = useField(streamFieldPath("paginator")); + const [field, , helpers] = useField(streamFieldPath("paginator")); const [pageSizeField] = useField(streamFieldPath("paginator.strategy.page_size")); const [, , pageSizeOptionHelpers] = useField(streamFieldPath("paginator.pageSizeOption")); @@ -23,8 +26,10 @@ export const PaginationSection: React.FC = ({ streamFiel helpers.setValue({ strategy: { type: "OffsetIncrement", + page_size: "", }, pageTokenOption: { + type: "RequestOption", inject_into: "request_parameter", }, }); @@ -48,12 +53,13 @@ export const PaginationSection: React.FC = ({ streamFiel ); const pageSizeOption = ( - label="Page size option" tooltip="Configures how the page size will be sent in requests to the source API" fieldPath={streamFieldPath("paginator.pageSizeOption")} initialValues={{ inject_into: "request_parameter", + type: "RequestOption", field_name: "", }} > diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/StreamSlicerSection.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/StreamSlicerSection.tsx index 230bc0c48255..e56e7825f113 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/StreamSlicerSection.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/StreamSlicerSection.tsx @@ -2,6 +2,8 @@ import { useField } from "formik"; import { ControlLabels } from "components/LabeledControl"; +import { RequestOption, SimpleRetrieverStreamSlicer } from "core/request/ConnectorManifest"; + import { timeDeltaRegex } from "../types"; import { BuilderCard } from "./BuilderCard"; import { BuilderField } from "./BuilderField"; @@ -15,7 +17,7 @@ interface StreamSlicerSectionProps { } export const StreamSlicerSection: React.FC = ({ streamFieldPath }) => { - const [field, , helpers] = useField(streamFieldPath("streamSlicer")); + const [field, , helpers] = useField(streamFieldPath("streamSlicer")); const handleToggle = (newToggleValue: boolean) => { if (newToggleValue) { @@ -65,12 +67,13 @@ export const StreamSlicerSection: React.FC = ({ stream label="Cursor field" tooltip="Field on record to use as the cursor" /> - label="Slice request option" tooltip="Optionally configures how the slice values will be sent in requests to the source API" fieldPath={streamFieldPath("streamSlicer.request_option")} initialValues={{ inject_into: "request_parameter", + type: "RequestOption", field_name: "", }} > @@ -127,12 +130,13 @@ export const StreamSlicerSection: React.FC = ({ stream tooltip="How many days before the start_datetime to read data for, e.g. 31d" optional /> - label="Start time request option" tooltip="Optionally configures how the start datetime will be sent in requests to the source API" fieldPath={streamFieldPath("streamSlicer.start_time_option")} initialValues={{ inject_into: "request_parameter", + type: "RequestOption", field_name: "", }} > @@ -142,12 +146,13 @@ export const StreamSlicerSection: React.FC = ({ stream excludeInjectIntoValues={["path"]} /> - label="End time request option" tooltip="Optionally configures how the end datetime will be sent in requests to the source API" fieldPath={streamFieldPath("streamSlicer.end_time_option")} initialValues={{ inject_into: "request_parameter", + type: "RequestOption", field_name: "", }} > diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/ToggleGroupField.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/ToggleGroupField.tsx index bd25d13ab793..865b48ad2aae 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/ToggleGroupField.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/ToggleGroupField.tsx @@ -6,20 +6,21 @@ import { CheckBox } from "components/ui/CheckBox"; import styles from "./ToggleGroupField.module.scss"; -interface ToggleGroupFieldProps { +interface ToggleGroupFieldProps { label: string; tooltip: string; fieldPath: string; - initialValues: unknown; + initialValues: T; } -export const ToggleGroupField: React.FC> = ({ +// eslint-disable-next-line react/function-component-definition +export function ToggleGroupField({ children, label, tooltip, fieldPath, initialValues, -}) => { +}: React.PropsWithChildren>) { const [field, , helpers] = useField(fieldPath); const enabled = field.value !== undefined; @@ -36,4 +37,4 @@ export const ToggleGroupField: React.FC{children} : labelComponent; -}; +} diff --git a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTestingPanel.tsx b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTestingPanel.tsx index f3799ed1fe46..6ac005ae807a 100644 --- a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTestingPanel.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTestingPanel.tsx @@ -6,10 +6,10 @@ import { Heading } from "components/ui/Heading"; import { Spinner } from "components/ui/Spinner"; import { Text } from "components/ui/Text"; -import { SourceDefinitionSpecificationDraft } from "core/domain/connector"; import { jsonSchemaToFormBlock } from "core/form/schemaToFormBlock"; import { buildYupFormForJsonSchema } from "core/form/schemaToYup"; import { StreamReadRequestBodyConfig } from "core/request/ConnectorBuilderClient"; +import { Spec } from "core/request/ConnectorManifest"; import { useConnectorBuilderState } from "services/connectorBuilder/ConnectorBuilderStateService"; import { links } from "utils/links"; @@ -20,13 +20,10 @@ import styles from "./StreamTestingPanel.module.scss"; const EMPTY_SCHEMA = {}; -function useConfigJsonErrors( - configJson: StreamReadRequestBodyConfig, - spec?: SourceDefinitionSpecificationDraft -): number { +function useConfigJsonErrors(configJson: StreamReadRequestBodyConfig, spec?: Spec): number { return useMemo(() => { try { - const jsonSchema = spec && spec.connectionSpecification ? spec.connectionSpecification : EMPTY_SCHEMA; + const jsonSchema = spec && spec.connection_specification ? spec.connection_specification : EMPTY_SCHEMA; const formFields = jsonSchemaToFormBlock(jsonSchema); const validationSchema = buildYupFormForJsonSchema(jsonSchema, formFields); validationSchema.validateSync(configJson, { abortEarly: false }); diff --git a/airbyte-webapp/src/components/connectorBuilder/types.ts b/airbyte-webapp/src/components/connectorBuilder/types.ts index 19feb2e54322..02cbb50f3702 100644 --- a/airbyte-webapp/src/components/connectorBuilder/types.ts +++ b/airbyte-webapp/src/components/connectorBuilder/types.ts @@ -13,9 +13,10 @@ import { NoAuth, SessionTokenAuthenticator, RequestOption, - DeclarativeOauth2Authenticator, + OAuthAuthenticator, DefaultPaginatorPaginationStrategy, SimpleRetrieverStreamSlicer, + HttpRequesterAuthenticator, } from "core/request/ConnectorManifest"; export interface BuilderFormInput { @@ -26,7 +27,7 @@ export interface BuilderFormInput { type BuilderFormAuthenticator = ( | NoAuth - | (Omit & { + | (Omit & { refresh_request_body: Array<[string, string]>; }) | ApiKeyAuthenticator @@ -46,6 +47,12 @@ export interface BuilderFormValues { streams: BuilderStream[]; } +export interface BuilderPaginator { + strategy: DefaultPaginatorPaginationStrategy; + pageTokenOption: RequestOption; + pageSizeOption?: RequestOption; +} + export interface BuilderStream { name: string; urlPath: string; @@ -57,11 +64,7 @@ export interface BuilderStream { requestHeaders: Array<[string, string]>; requestBody: Array<[string, string]>; }; - paginator?: { - strategy: DefaultPaginatorPaginationStrategy; - pageTokenOption: RequestOption; - pageSizeOption?: RequestOption; - }; + paginator?: BuilderPaginator; streamSlicer?: SimpleRetrieverStreamSlicer; } @@ -375,7 +378,7 @@ export const builderFormValidationSchema = yup.object().shape({ function builderFormAuthenticatorToAuthenticator( globalSettings: BuilderFormValues["global"] -): HttpRequesterAllOfAuthenticator { +): HttpRequesterAuthenticator { if (globalSettings.authenticator.type === "OAuthAuthenticator") { return { ...globalSettings.authenticator, @@ -388,7 +391,7 @@ function builderFormAuthenticatorToAuthenticator( api_url: globalSettings.urlBase, }; } - return globalSettings.authenticator as HttpRequesterAllOfAuthenticator; + return globalSettings.authenticator as HttpRequesterAuthenticator; } export const convertToManifest = (values: BuilderFormValues): ConnectorManifest => { diff --git a/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx b/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx index 8c83f1b853aa..356afdf64dee 100644 --- a/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx +++ b/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx @@ -7,10 +7,11 @@ import { useLocalStorage } from "react-use"; import { BuilderFormValues, convertToManifest, DEFAULT_BUILDER_FORM_VALUES } from "components/connectorBuilder/types"; import { StreamReadRequestBodyConfig, StreamsListReadStreamsItem } from "core/request/ConnectorBuilderClient"; +import { ConnectorManifest } from "core/request/ConnectorManifest"; import { useListStreams } from "./ConnectorBuilderApiService"; -const DEFAULT_JSON_MANIFEST_VALUES: PatchedConnectorManifest = { +const DEFAULT_JSON_MANIFEST_VALUES: ConnectorManifest = { version: "0.1.0", type: "DeclarativeSource", check: { @@ -36,7 +37,7 @@ interface Context { configJson: StreamReadRequestBodyConfig; editorView: EditorView; setBuilderFormValues: (values: BuilderFormValues, isInvalid: boolean) => void; - setJsonManifest: (jsonValue: PatchedConnectorManifest) => void; + setJsonManifest: (jsonValue: ConnectorManifest) => void; setYamlEditorIsMounted: (value: boolean) => void; setYamlIsValid: (value: boolean) => void; setTestStreamIndex: (streamIndex: number) => void; @@ -72,7 +73,7 @@ export const ConnectorBuilderStateProvider: React.FC( + const [jsonManifest, setJsonManifest] = useLocalStorage( "connectorBuilderJsonManifest", DEFAULT_JSON_MANIFEST_VALUES );