🪟🔧 Connector builder: use new lowcode manifest (#20715)

* use new manifest yaml * Update airbyte-webapp/src/components/connectorBuilder/types.ts Co-authored-by: Lake Mossman <[email protected]> * use updated manifest types Co-authored-by: Lake Mossman <[email protected]>
airbytehq · Dec 22, 2022 · a9e1d09 · a9e1d09
1 parent ae9a229
commit a9e1d09
Show file tree

Hide file tree

Showing 10 changed files with 83 additions and 55 deletions.
diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/InjectRequestOptionFields.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/InjectRequestOptionFields.tsx
@@ -30,7 +30,7 @@ export const InjectRequestOptionFields: React.FC<InjectRequestOptionFieldsProps>
         }
         onChange={(newValue) => {
           if (newValue === "path") {
-            helpers.setValue({ inject_into: newValue, field_name: undefined });
+            helpers.setValue({ inject_into: newValue, field_name: undefined, type: "RequestOption" });
           }
         }}
         label="Inject into"

diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/PaginationSection.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/PaginationSection.tsx
@@ -3,6 +3,9 @@ import { useField } from "formik";
 import GroupControls from "components/GroupControls";
 import { ControlLabels } from "components/LabeledControl";
 
+import { RequestOption } from "core/request/ConnectorManifest";
+
+import { BuilderPaginator } from "../types";
 import { BuilderCard } from "./BuilderCard";
 import { BuilderField } from "./BuilderField";
 import { BuilderOneOf } from "./BuilderOneOf";
@@ -14,7 +17,7 @@ interface PaginationSectionProps {
 }
 
 export const PaginationSection: React.FC<PaginationSectionProps> = ({ streamFieldPath }) => {
-  const [field, , helpers] = useField(streamFieldPath("paginator"));
+  const [field, , helpers] = useField<BuilderPaginator | undefined>(streamFieldPath("paginator"));
   const [pageSizeField] = useField(streamFieldPath("paginator.strategy.page_size"));
   const [, , pageSizeOptionHelpers] = useField(streamFieldPath("paginator.pageSizeOption"));
 
@@ -23,8 +26,10 @@ export const PaginationSection: React.FC<PaginationSectionProps> = ({ streamFiel
       helpers.setValue({
         strategy: {
           type: "OffsetIncrement",
+          page_size: "",
         },
         pageTokenOption: {
+          type: "RequestOption",
           inject_into: "request_parameter",
         },
       });
@@ -48,12 +53,13 @@ export const PaginationSection: React.FC<PaginationSectionProps> = ({ streamFiel
   );
 
   const pageSizeOption = (
-    <ToggleGroupField
+    <ToggleGroupField<RequestOption>
       label="Page size option"
       tooltip="Configures how the page size will be sent in requests to the source API"
       fieldPath={streamFieldPath("paginator.pageSizeOption")}
       initialValues={{
         inject_into: "request_parameter",
+        type: "RequestOption",
         field_name: "",
       }}
     >

diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/StreamSlicerSection.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/StreamSlicerSection.tsx
@@ -2,6 +2,8 @@ import { useField } from "formik";
 
 import { ControlLabels } from "components/LabeledControl";
 
+import { RequestOption, SimpleRetrieverStreamSlicer } from "core/request/ConnectorManifest";
+
 import { timeDeltaRegex } from "../types";
 import { BuilderCard } from "./BuilderCard";
 import { BuilderField } from "./BuilderField";
@@ -15,7 +17,7 @@ interface StreamSlicerSectionProps {
 }
 
 export const StreamSlicerSection: React.FC<StreamSlicerSectionProps> = ({ streamFieldPath }) => {
-  const [field, , helpers] = useField(streamFieldPath("streamSlicer"));
+  const [field, , helpers] = useField<SimpleRetrieverStreamSlicer | undefined>(streamFieldPath("streamSlicer"));
 
   const handleToggle = (newToggleValue: boolean) => {
     if (newToggleValue) {
@@ -65,12 +67,13 @@ export const StreamSlicerSection: React.FC<StreamSlicerSectionProps> = ({ stream
                   label="Cursor field"
                   tooltip="Field on record to use as the cursor"
                 />
-                <ToggleGroupField
+                <ToggleGroupField<RequestOption>
                   label="Slice request option"
                   tooltip="Optionally configures how the slice values will be sent in requests to the source API"
                   fieldPath={streamFieldPath("streamSlicer.request_option")}
                   initialValues={{
                     inject_into: "request_parameter",
+                    type: "RequestOption",
                     field_name: "",
                   }}
                 >
@@ -127,12 +130,13 @@ export const StreamSlicerSection: React.FC<StreamSlicerSectionProps> = ({ stream
                     tooltip="How many days before the start_datetime to read data for, e.g. 31d"
                     optional
                   />
-                  <ToggleGroupField
+                  <ToggleGroupField<RequestOption>
                     label="Start time request option"
                     tooltip="Optionally configures how the start datetime will be sent in requests to the source API"
                     fieldPath={streamFieldPath("streamSlicer.start_time_option")}
                     initialValues={{
                       inject_into: "request_parameter",
+                      type: "RequestOption",
                       field_name: "",
                     }}
                   >
@@ -142,12 +146,13 @@ export const StreamSlicerSection: React.FC<StreamSlicerSectionProps> = ({ stream
                       excludeInjectIntoValues={["path"]}
                     />
                   </ToggleGroupField>
-                  <ToggleGroupField
+                  <ToggleGroupField<RequestOption>
                     label="End time request option"
                     tooltip="Optionally configures how the end datetime will be sent in requests to the source API"
                     fieldPath={streamFieldPath("streamSlicer.end_time_option")}
                     initialValues={{
                       inject_into: "request_parameter",
+                      type: "RequestOption",
                       field_name: "",
                     }}
                   >

diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/ToggleGroupField.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/ToggleGroupField.tsx
@@ -6,20 +6,21 @@ import { CheckBox } from "components/ui/CheckBox";
 
 import styles from "./ToggleGroupField.module.scss";
 
-interface ToggleGroupFieldProps {
+interface ToggleGroupFieldProps<T> {
   label: string;
   tooltip: string;
   fieldPath: string;
-  initialValues: unknown;
+  initialValues: T;
 }
 
-export const ToggleGroupField: React.FC<React.PropsWithChildren<ToggleGroupFieldProps>> = ({
+// eslint-disable-next-line react/function-component-definition
+export function ToggleGroupField<T>({
   children,
   label,
   tooltip,
   fieldPath,
   initialValues,
-}) => {
+}: React.PropsWithChildren<ToggleGroupFieldProps<T>>) {
   const [field, , helpers] = useField(fieldPath);
   const enabled = field.value !== undefined;
 
@@ -36,4 +37,4 @@ export const ToggleGroupField: React.FC<React.PropsWithChildren<ToggleGroupField
   );
 
   return enabled ? <GroupControls label={labelComponent}>{children}</GroupControls> : labelComponent;
-};
+}
diff --git a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/ConfigMenu.tsx b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/ConfigMenu.tsx
@@ -1,5 +1,6 @@
 import { faClose, faUser } from "@fortawesome/free-solid-svg-icons";
 import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
+import { useMemo } from "react";
 import { FormattedMessage, useIntl } from "react-intl";
 import { useLocalStorage } from "react-use";
 
@@ -9,6 +10,7 @@ import { Modal, ModalBody } from "components/ui/Modal";
 import { NumberBadge } from "components/ui/NumberBadge";
 import { Tooltip } from "components/ui/Tooltip";
 
+import { SourceDefinitionSpecificationDraft } from "core/domain/connector";
 import { StreamReadRequestBodyConfig } from "core/request/ConnectorBuilderClient";
 import { useConnectorBuilderState } from "services/connectorBuilder/ConnectorBuilderStateService";
 import { ConnectorForm } from "views/Connector/ConnectorForm";
@@ -34,6 +36,17 @@ export const ConfigMenu: React.FC<ConfigMenuProps> = ({ className, configJsonErr
     setIsOpen(false);
   };
 
+  const connectorDefinitionSpecification: SourceDefinitionSpecificationDraft | undefined = useMemo(
+    () =>
+      jsonManifest.spec
+        ? {
+            documentationUrl: jsonManifest.spec.documentation_url,
+            connectionSpecification: jsonManifest.spec.connection_specification,
+          }
+        : undefined,
+    [jsonManifest]
+  );
+
   return (
     <>
       <Tooltip
@@ -64,7 +77,7 @@ export const ConfigMenu: React.FC<ConfigMenuProps> = ({ className, configJsonErr
           <FormattedMessage id="connectorBuilder.inputsNoSpecYAMLTooltip" />
         )}
       </Tooltip>
-      {isOpen && jsonManifest.spec && (
+      {isOpen && connectorDefinitionSpecification && (
         <Modal
           size="lg"
           onClose={() => setIsOpen(false)}
@@ -93,7 +106,7 @@ export const ConfigMenu: React.FC<ConfigMenuProps> = ({ className, configJsonErr
                   formType="source"
                   bodyClassName={styles.formContent}
                   footerClassName={styles.inputFormModalFooter}
-                  selectedConnectorDefinitionSpecification={jsonManifest.spec}
+                  selectedConnectorDefinitionSpecification={connectorDefinitionSpecification}
                   formValues={{ connectionConfiguration: configJson }}
                   onSubmit={async (values) => {
                     setConfigJson(values.connectionConfiguration as StreamReadRequestBodyConfig);

diff --git a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTestingPanel.tsx b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTestingPanel.tsx
@@ -6,10 +6,10 @@ import { Heading } from "components/ui/Heading";
 import { Spinner } from "components/ui/Spinner";
 import { Text } from "components/ui/Text";
 
-import { SourceDefinitionSpecificationDraft } from "core/domain/connector";
 import { jsonSchemaToFormBlock } from "core/form/schemaToFormBlock";
 import { buildYupFormForJsonSchema } from "core/form/schemaToYup";
 import { StreamReadRequestBodyConfig } from "core/request/ConnectorBuilderClient";
+import { Spec } from "core/request/ConnectorManifest";
 import { useConnectorBuilderState } from "services/connectorBuilder/ConnectorBuilderStateService";
 import { links } from "utils/links";
 
@@ -20,13 +20,10 @@ import styles from "./StreamTestingPanel.module.scss";
 
 const EMPTY_SCHEMA = {};
 
-function useConfigJsonErrors(
-  configJson: StreamReadRequestBodyConfig,
-  spec?: SourceDefinitionSpecificationDraft
-): number {
+function useConfigJsonErrors(configJson: StreamReadRequestBodyConfig, spec?: Spec): number {
   return useMemo(() => {
     try {
-      const jsonSchema = spec && spec.connectionSpecification ? spec.connectionSpecification : EMPTY_SCHEMA;
+      const jsonSchema = spec && spec.connection_specification ? spec.connection_specification : EMPTY_SCHEMA;
       const formFields = jsonSchemaToFormBlock(jsonSchema);
       const validationSchema = buildYupFormForJsonSchema(jsonSchema, formFields);
       validationSchema.validateSync(configJson, { abortEarly: false });

diff --git a/airbyte-webapp/src/components/connectorBuilder/types.ts b/airbyte-webapp/src/components/connectorBuilder/types.ts
@@ -1,21 +1,22 @@
 import { JSONSchema7 } from "json-schema";
 import * as yup from "yup";
 
-import { SourceDefinitionSpecificationDraft } from "core/domain/connector";
-import { PatchedConnectorManifest } from "core/domain/connectorBuilder/PatchedConnectorManifest";
 import { AirbyteJSONSchema } from "core/jsonSchema/types";
 import {
+  ConnectorManifest,
+  InterpolatedRequestOptionsProvider,
+  Spec,
   ApiKeyAuthenticator,
   BasicHttpAuthenticator,
   BearerAuthenticator,
-  DeclarativeOauth2AuthenticatorAllOf,
   DeclarativeStream,
-  HttpRequesterAllOfAuthenticator,
   NoAuth,
   SessionTokenAuthenticator,
-  DefaultPaginatorAllOfPaginationStrategy,
   RequestOption,
-  SimpleRetrieverAllOfStreamSlicer,
+  OAuthAuthenticator,
+  DefaultPaginatorPaginationStrategy,
+  SimpleRetrieverStreamSlicer,
+  HttpRequesterAuthenticator,
 } from "core/request/ConnectorManifest";
 
 export interface BuilderFormInput {
@@ -26,7 +27,7 @@ export interface BuilderFormInput {
 
 type BuilderFormAuthenticator = (
   | NoAuth
-  | (Omit<DeclarativeOauth2AuthenticatorAllOf, "refresh_request_body"> & {
+  | (Omit<OAuthAuthenticator, "refresh_request_body"> & {
       refresh_request_body: Array<[string, string]>;
     })
   | ApiKeyAuthenticator
@@ -46,6 +47,12 @@ export interface BuilderFormValues {
   streams: BuilderStream[];
 }
 
+export interface BuilderPaginator {
+  strategy: DefaultPaginatorPaginationStrategy;
+  pageTokenOption: RequestOption;
+  pageSizeOption?: RequestOption;
+}
+
 export interface BuilderStream {
   name: string;
   urlPath: string;
@@ -57,12 +64,8 @@ export interface BuilderStream {
     requestHeaders: Array<[string, string]>;
     requestBody: Array<[string, string]>;
   };
-  paginator?: {
-    strategy: DefaultPaginatorAllOfPaginationStrategy;
-    pageTokenOption: RequestOption;
-    pageSizeOption?: RequestOption;
-  };
-  streamSlicer?: SimpleRetrieverAllOfStreamSlicer;
+  paginator?: BuilderPaginator;
+  streamSlicer?: SimpleRetrieverStreamSlicer;
 }
 
 export const DEFAULT_BUILDER_FORM_VALUES: BuilderFormValues = {
@@ -375,7 +378,7 @@ export const builderFormValidationSchema = yup.object().shape({
 
 function builderFormAuthenticatorToAuthenticator(
   globalSettings: BuilderFormValues["global"]
-): HttpRequesterAllOfAuthenticator {
+): HttpRequesterAuthenticator {
   if (globalSettings.authenticator.type === "OAuthAuthenticator") {
     return {
       ...globalSettings.authenticator,
@@ -388,34 +391,40 @@ function builderFormAuthenticatorToAuthenticator(
       api_url: globalSettings.urlBase,
     };
   }
-  return globalSettings.authenticator as HttpRequesterAllOfAuthenticator;
+  return globalSettings.authenticator as HttpRequesterAuthenticator;
 }
 
-export const convertToManifest = (values: BuilderFormValues): PatchedConnectorManifest => {
+export const convertToManifest = (values: BuilderFormValues): ConnectorManifest => {
   const manifestStreams: DeclarativeStream[] = values.streams.map((stream) => {
     return {
+      type: "DeclarativeStream",
       name: stream.name,
       primary_key: stream.primaryKey,
       retriever: {
+        type: "SimpleRetriever",
         name: stream.name,
         primary_key: stream.primaryKey,
         requester: {
+          type: "HttpRequester",
           name: stream.name,
           url_base: values.global?.urlBase,
           path: stream.urlPath,
           request_options_provider: {
+            // TODO can't declare type here because the server will error out, but the types dictate it is needed. Fix here once server is fixed.
+            // type: "InterpolatedRequestOptionsProvider",
             request_parameters: Object.fromEntries(stream.requestOptions.requestParameters),
             request_headers: Object.fromEntries(stream.requestOptions.requestHeaders),
             request_body_json: Object.fromEntries(stream.requestOptions.requestBody),
-          },
+          } as InterpolatedRequestOptionsProvider,
           authenticator: builderFormAuthenticatorToAuthenticator(values.global),
           // TODO: remove these empty "config" values once they are no longer required in the connector manifest JSON schema
           config: {},
         },
         record_selector: {
+          type: "RecordSelector",
           extractor: {
+            type: "DpathExtractor",
             field_pointer: stream.fieldPointer,
-            config: {},
           },
         },
         paginator: stream.paginator
@@ -436,7 +445,6 @@ export const convertToManifest = (values: BuilderFormValues): PatchedConnectorMa
         stream_slicer: stream.streamSlicer,
         config: {},
       },
-      config: {},
     };
   });
 
@@ -450,13 +458,17 @@ export const convertToManifest = (values: BuilderFormValues): PatchedConnectorMa
     additionalProperties: true,
   };
 
-  const spec: SourceDefinitionSpecificationDraft = {
-    connectionSpecification: specSchema,
+  const spec: Spec = {
+    connection_specification: specSchema,
+    documentation_url: "",
+    type: "Spec",
   };
 
   return {
     version: "0.1.0",
+    type: "DeclarativeSource",
     check: {
+      type: "CheckStream",
       stream_names: [],
     },
     streams: manifestStreams,

diff --git a/airbyte-webapp/src/core/domain/connectorBuilder/PatchedConnectorManifest.ts b/airbyte-webapp/src/core/domain/connectorBuilder/PatchedConnectorManifest.ts