diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/InjectRequestOptionFields.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/InjectRequestOptionFields.tsx index 7f556b44cc0e..3d8607eb85f9 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/InjectRequestOptionFields.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/InjectRequestOptionFields.tsx @@ -30,7 +30,7 @@ export const InjectRequestOptionFields: React.FC } onChange={(newValue) => { if (newValue === "path") { - helpers.setValue({ inject_into: newValue, field_name: undefined }); + helpers.setValue({ inject_into: newValue, field_name: undefined, type: "RequestOption" }); } }} label="Inject into" diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/PaginationSection.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/PaginationSection.tsx index 4082e34cf04e..0a6878c28fa0 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/PaginationSection.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/PaginationSection.tsx @@ -3,6 +3,9 @@ import { useField } from "formik"; import GroupControls from "components/GroupControls"; import { ControlLabels } from "components/LabeledControl"; +import { RequestOption } from "core/request/ConnectorManifest"; + +import { BuilderPaginator } from "../types"; import { BuilderCard } from "./BuilderCard"; import { BuilderField } from "./BuilderField"; import { BuilderOneOf } from "./BuilderOneOf"; @@ -14,7 +17,7 @@ interface PaginationSectionProps { } export const PaginationSection: React.FC = ({ streamFieldPath }) => { - const [field, , helpers] = useField(streamFieldPath("paginator")); + const [field, , helpers] = useField(streamFieldPath("paginator")); const [pageSizeField] = useField(streamFieldPath("paginator.strategy.page_size")); const [, , pageSizeOptionHelpers] = useField(streamFieldPath("paginator.pageSizeOption")); @@ -23,8 +26,10 @@ export const PaginationSection: React.FC = ({ streamFiel helpers.setValue({ strategy: { type: "OffsetIncrement", + page_size: "", }, pageTokenOption: { + type: "RequestOption", inject_into: "request_parameter", }, }); @@ -48,12 +53,13 @@ export const PaginationSection: React.FC = ({ streamFiel ); const pageSizeOption = ( - label="Page size option" tooltip="Configures how the page size will be sent in requests to the source API" fieldPath={streamFieldPath("paginator.pageSizeOption")} initialValues={{ inject_into: "request_parameter", + type: "RequestOption", field_name: "", }} > diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/StreamSlicerSection.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/StreamSlicerSection.tsx index 230bc0c48255..e56e7825f113 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/StreamSlicerSection.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/StreamSlicerSection.tsx @@ -2,6 +2,8 @@ import { useField } from "formik"; import { ControlLabels } from "components/LabeledControl"; +import { RequestOption, SimpleRetrieverStreamSlicer } from "core/request/ConnectorManifest"; + import { timeDeltaRegex } from "../types"; import { BuilderCard } from "./BuilderCard"; import { BuilderField } from "./BuilderField"; @@ -15,7 +17,7 @@ interface StreamSlicerSectionProps { } export const StreamSlicerSection: React.FC = ({ streamFieldPath }) => { - const [field, , helpers] = useField(streamFieldPath("streamSlicer")); + const [field, , helpers] = useField(streamFieldPath("streamSlicer")); const handleToggle = (newToggleValue: boolean) => { if (newToggleValue) { @@ -65,12 +67,13 @@ export const StreamSlicerSection: React.FC = ({ stream label="Cursor field" tooltip="Field on record to use as the cursor" /> - label="Slice request option" tooltip="Optionally configures how the slice values will be sent in requests to the source API" fieldPath={streamFieldPath("streamSlicer.request_option")} initialValues={{ inject_into: "request_parameter", + type: "RequestOption", field_name: "", }} > @@ -127,12 +130,13 @@ export const StreamSlicerSection: React.FC = ({ stream tooltip="How many days before the start_datetime to read data for, e.g. 31d" optional /> - label="Start time request option" tooltip="Optionally configures how the start datetime will be sent in requests to the source API" fieldPath={streamFieldPath("streamSlicer.start_time_option")} initialValues={{ inject_into: "request_parameter", + type: "RequestOption", field_name: "", }} > @@ -142,12 +146,13 @@ export const StreamSlicerSection: React.FC = ({ stream excludeInjectIntoValues={["path"]} /> - label="End time request option" tooltip="Optionally configures how the end datetime will be sent in requests to the source API" fieldPath={streamFieldPath("streamSlicer.end_time_option")} initialValues={{ inject_into: "request_parameter", + type: "RequestOption", field_name: "", }} > diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/ToggleGroupField.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/ToggleGroupField.tsx index bd25d13ab793..865b48ad2aae 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/ToggleGroupField.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/ToggleGroupField.tsx @@ -6,20 +6,21 @@ import { CheckBox } from "components/ui/CheckBox"; import styles from "./ToggleGroupField.module.scss"; -interface ToggleGroupFieldProps { +interface ToggleGroupFieldProps { label: string; tooltip: string; fieldPath: string; - initialValues: unknown; + initialValues: T; } -export const ToggleGroupField: React.FC> = ({ +// eslint-disable-next-line react/function-component-definition +export function ToggleGroupField({ children, label, tooltip, fieldPath, initialValues, -}) => { +}: React.PropsWithChildren>) { const [field, , helpers] = useField(fieldPath); const enabled = field.value !== undefined; @@ -36,4 +37,4 @@ export const ToggleGroupField: React.FC{children} : labelComponent; -}; +} diff --git a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/ConfigMenu.tsx b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/ConfigMenu.tsx index b2b3f224d96b..730bdfc44d60 100644 --- a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/ConfigMenu.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/ConfigMenu.tsx @@ -1,5 +1,6 @@ import { faClose, faUser } from "@fortawesome/free-solid-svg-icons"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; +import { useMemo } from "react"; import { FormattedMessage, useIntl } from "react-intl"; import { useLocalStorage } from "react-use"; @@ -9,6 +10,7 @@ import { Modal, ModalBody } from "components/ui/Modal"; import { NumberBadge } from "components/ui/NumberBadge"; import { Tooltip } from "components/ui/Tooltip"; +import { SourceDefinitionSpecificationDraft } from "core/domain/connector"; import { StreamReadRequestBodyConfig } from "core/request/ConnectorBuilderClient"; import { useConnectorBuilderState } from "services/connectorBuilder/ConnectorBuilderStateService"; import { ConnectorForm } from "views/Connector/ConnectorForm"; @@ -34,6 +36,17 @@ export const ConfigMenu: React.FC = ({ className, configJsonErr setIsOpen(false); }; + const connectorDefinitionSpecification: SourceDefinitionSpecificationDraft | undefined = useMemo( + () => + jsonManifest.spec + ? { + documentationUrl: jsonManifest.spec.documentation_url, + connectionSpecification: jsonManifest.spec.connection_specification, + } + : undefined, + [jsonManifest] + ); + return ( <> = ({ className, configJsonErr )} - {isOpen && jsonManifest.spec && ( + {isOpen && connectorDefinitionSpecification && ( setIsOpen(false)} @@ -93,7 +106,7 @@ export const ConfigMenu: React.FC = ({ className, configJsonErr formType="source" bodyClassName={styles.formContent} footerClassName={styles.inputFormModalFooter} - selectedConnectorDefinitionSpecification={jsonManifest.spec} + selectedConnectorDefinitionSpecification={connectorDefinitionSpecification} formValues={{ connectionConfiguration: configJson }} onSubmit={async (values) => { setConfigJson(values.connectionConfiguration as StreamReadRequestBodyConfig); diff --git a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTestingPanel.tsx b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTestingPanel.tsx index f3799ed1fe46..6ac005ae807a 100644 --- a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTestingPanel.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTestingPanel.tsx @@ -6,10 +6,10 @@ import { Heading } from "components/ui/Heading"; import { Spinner } from "components/ui/Spinner"; import { Text } from "components/ui/Text"; -import { SourceDefinitionSpecificationDraft } from "core/domain/connector"; import { jsonSchemaToFormBlock } from "core/form/schemaToFormBlock"; import { buildYupFormForJsonSchema } from "core/form/schemaToYup"; import { StreamReadRequestBodyConfig } from "core/request/ConnectorBuilderClient"; +import { Spec } from "core/request/ConnectorManifest"; import { useConnectorBuilderState } from "services/connectorBuilder/ConnectorBuilderStateService"; import { links } from "utils/links"; @@ -20,13 +20,10 @@ import styles from "./StreamTestingPanel.module.scss"; const EMPTY_SCHEMA = {}; -function useConfigJsonErrors( - configJson: StreamReadRequestBodyConfig, - spec?: SourceDefinitionSpecificationDraft -): number { +function useConfigJsonErrors(configJson: StreamReadRequestBodyConfig, spec?: Spec): number { return useMemo(() => { try { - const jsonSchema = spec && spec.connectionSpecification ? spec.connectionSpecification : EMPTY_SCHEMA; + const jsonSchema = spec && spec.connection_specification ? spec.connection_specification : EMPTY_SCHEMA; const formFields = jsonSchemaToFormBlock(jsonSchema); const validationSchema = buildYupFormForJsonSchema(jsonSchema, formFields); validationSchema.validateSync(configJson, { abortEarly: false }); diff --git a/airbyte-webapp/src/components/connectorBuilder/types.ts b/airbyte-webapp/src/components/connectorBuilder/types.ts index f56dc225e156..02cbb50f3702 100644 --- a/airbyte-webapp/src/components/connectorBuilder/types.ts +++ b/airbyte-webapp/src/components/connectorBuilder/types.ts @@ -1,21 +1,22 @@ import { JSONSchema7 } from "json-schema"; import * as yup from "yup"; -import { SourceDefinitionSpecificationDraft } from "core/domain/connector"; -import { PatchedConnectorManifest } from "core/domain/connectorBuilder/PatchedConnectorManifest"; import { AirbyteJSONSchema } from "core/jsonSchema/types"; import { + ConnectorManifest, + InterpolatedRequestOptionsProvider, + Spec, ApiKeyAuthenticator, BasicHttpAuthenticator, BearerAuthenticator, - DeclarativeOauth2AuthenticatorAllOf, DeclarativeStream, - HttpRequesterAllOfAuthenticator, NoAuth, SessionTokenAuthenticator, - DefaultPaginatorAllOfPaginationStrategy, RequestOption, - SimpleRetrieverAllOfStreamSlicer, + OAuthAuthenticator, + DefaultPaginatorPaginationStrategy, + SimpleRetrieverStreamSlicer, + HttpRequesterAuthenticator, } from "core/request/ConnectorManifest"; export interface BuilderFormInput { @@ -26,7 +27,7 @@ export interface BuilderFormInput { type BuilderFormAuthenticator = ( | NoAuth - | (Omit & { + | (Omit & { refresh_request_body: Array<[string, string]>; }) | ApiKeyAuthenticator @@ -46,6 +47,12 @@ export interface BuilderFormValues { streams: BuilderStream[]; } +export interface BuilderPaginator { + strategy: DefaultPaginatorPaginationStrategy; + pageTokenOption: RequestOption; + pageSizeOption?: RequestOption; +} + export interface BuilderStream { name: string; urlPath: string; @@ -57,12 +64,8 @@ export interface BuilderStream { requestHeaders: Array<[string, string]>; requestBody: Array<[string, string]>; }; - paginator?: { - strategy: DefaultPaginatorAllOfPaginationStrategy; - pageTokenOption: RequestOption; - pageSizeOption?: RequestOption; - }; - streamSlicer?: SimpleRetrieverAllOfStreamSlicer; + paginator?: BuilderPaginator; + streamSlicer?: SimpleRetrieverStreamSlicer; } export const DEFAULT_BUILDER_FORM_VALUES: BuilderFormValues = { @@ -375,7 +378,7 @@ export const builderFormValidationSchema = yup.object().shape({ function builderFormAuthenticatorToAuthenticator( globalSettings: BuilderFormValues["global"] -): HttpRequesterAllOfAuthenticator { +): HttpRequesterAuthenticator { if (globalSettings.authenticator.type === "OAuthAuthenticator") { return { ...globalSettings.authenticator, @@ -388,34 +391,40 @@ function builderFormAuthenticatorToAuthenticator( api_url: globalSettings.urlBase, }; } - return globalSettings.authenticator as HttpRequesterAllOfAuthenticator; + return globalSettings.authenticator as HttpRequesterAuthenticator; } -export const convertToManifest = (values: BuilderFormValues): PatchedConnectorManifest => { +export const convertToManifest = (values: BuilderFormValues): ConnectorManifest => { const manifestStreams: DeclarativeStream[] = values.streams.map((stream) => { return { + type: "DeclarativeStream", name: stream.name, primary_key: stream.primaryKey, retriever: { + type: "SimpleRetriever", name: stream.name, primary_key: stream.primaryKey, requester: { + type: "HttpRequester", name: stream.name, url_base: values.global?.urlBase, path: stream.urlPath, request_options_provider: { + // TODO can't declare type here because the server will error out, but the types dictate it is needed. Fix here once server is fixed. + // type: "InterpolatedRequestOptionsProvider", request_parameters: Object.fromEntries(stream.requestOptions.requestParameters), request_headers: Object.fromEntries(stream.requestOptions.requestHeaders), request_body_json: Object.fromEntries(stream.requestOptions.requestBody), - }, + } as InterpolatedRequestOptionsProvider, authenticator: builderFormAuthenticatorToAuthenticator(values.global), // TODO: remove these empty "config" values once they are no longer required in the connector manifest JSON schema config: {}, }, record_selector: { + type: "RecordSelector", extractor: { + type: "DpathExtractor", field_pointer: stream.fieldPointer, - config: {}, }, }, paginator: stream.paginator @@ -436,7 +445,6 @@ export const convertToManifest = (values: BuilderFormValues): PatchedConnectorMa stream_slicer: stream.streamSlicer, config: {}, }, - config: {}, }; }); @@ -450,13 +458,17 @@ export const convertToManifest = (values: BuilderFormValues): PatchedConnectorMa additionalProperties: true, }; - const spec: SourceDefinitionSpecificationDraft = { - connectionSpecification: specSchema, + const spec: Spec = { + connection_specification: specSchema, + documentation_url: "", + type: "Spec", }; return { version: "0.1.0", + type: "DeclarativeSource", check: { + type: "CheckStream", stream_names: [], }, streams: manifestStreams, diff --git a/airbyte-webapp/src/core/domain/connectorBuilder/PatchedConnectorManifest.ts b/airbyte-webapp/src/core/domain/connectorBuilder/PatchedConnectorManifest.ts deleted file mode 100644 index 2e8dd79b575f..000000000000 --- a/airbyte-webapp/src/core/domain/connectorBuilder/PatchedConnectorManifest.ts +++ /dev/null @@ -1,8 +0,0 @@ -import { SourceDefinitionSpecificationDraft } from "core/domain/connector"; - -import { ConnectorManifest } from "../../request/ConnectorManifest"; - -// Patching this type as required until the upstream schema is updated -export interface PatchedConnectorManifest extends ConnectorManifest { - spec?: SourceDefinitionSpecificationDraft; -} diff --git a/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx b/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx index 435c7ebe4e43..356afdf64dee 100644 --- a/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx +++ b/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx @@ -6,14 +6,16 @@ import { useLocalStorage } from "react-use"; import { BuilderFormValues, convertToManifest, DEFAULT_BUILDER_FORM_VALUES } from "components/connectorBuilder/types"; -import { PatchedConnectorManifest } from "core/domain/connectorBuilder/PatchedConnectorManifest"; import { StreamReadRequestBodyConfig, StreamsListReadStreamsItem } from "core/request/ConnectorBuilderClient"; +import { ConnectorManifest } from "core/request/ConnectorManifest"; import { useListStreams } from "./ConnectorBuilderApiService"; -const DEFAULT_JSON_MANIFEST_VALUES: PatchedConnectorManifest = { +const DEFAULT_JSON_MANIFEST_VALUES: ConnectorManifest = { version: "0.1.0", + type: "DeclarativeSource", check: { + type: "CheckStream", stream_names: [], }, streams: [], @@ -24,7 +26,7 @@ export type BuilderView = "global" | "inputs" | number; interface Context { builderFormValues: BuilderFormValues; - jsonManifest: PatchedConnectorManifest; + jsonManifest: ConnectorManifest; yamlManifest: string; yamlEditorIsMounted: boolean; yamlIsValid: boolean; @@ -35,7 +37,7 @@ interface Context { configJson: StreamReadRequestBodyConfig; editorView: EditorView; setBuilderFormValues: (values: BuilderFormValues, isInvalid: boolean) => void; - setJsonManifest: (jsonValue: PatchedConnectorManifest) => void; + setJsonManifest: (jsonValue: ConnectorManifest) => void; setYamlEditorIsMounted: (value: boolean) => void; setYamlIsValid: (value: boolean) => void; setTestStreamIndex: (streamIndex: number) => void; @@ -71,7 +73,7 @@ export const ConnectorBuilderStateProvider: React.FC( + const [jsonManifest, setJsonManifest] = useLocalStorage( "connectorBuilderJsonManifest", DEFAULT_JSON_MANIFEST_VALUES ); diff --git a/airbyte-webapp/src/services/connectorBuilder/connector_manifest_openapi.yaml b/airbyte-webapp/src/services/connectorBuilder/connector_manifest_openapi.yaml index 89fe28623293..690df8ae8fe4 100644 --- a/airbyte-webapp/src/services/connectorBuilder/connector_manifest_openapi.yaml +++ b/airbyte-webapp/src/services/connectorBuilder/connector_manifest_openapi.yaml @@ -6,4 +6,4 @@ paths: {} components: schemas: ConnectorManifest: - $ref: "../../../../airbyte-cdk/python/airbyte_cdk/sources/declarative/config_component_schema.json" + $ref: "../../../../airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_component_schema.yaml"