[Connector Builder]Add ability to convert from YAML manifest to UI (#…

…21142)

* save

* save more progress

* try setting values directly

* toggle editor

* fix primary key

* enforce consistency in name and primary key

* refactor conversion method to be more readable

* save progress

* allow custom input keys to be used for inferred auth values

* fix isMatch bug and remove console logs

* fix type issues with reflect

* properly handle undefined

* format schema and gracefully handle non-inline schemas

* verify no custom components

* refactor and fix request options type

* rest of refactor

* move manifest to builder form conversion logic into its own file, and handle inferred input overrides properly

* convert substream slicers

* restore warning modal for switching back to UI

* remove console logs

* remove unneeded traceback filtering

* set http method when converting to manifest

* remove commented import

* add unsupported fields to builder form values

* save check stream values from manifest

* save progress

* add more tests

* save record filter in unsupported fields

* use type coersion instead of yaml strings

airbyte-webapp/src/components/common/ConfirmationModal/ConfirmationModal.tsx

@@ -11,6 +11,7 @@ export interface ConfirmationModalProps {
   onClose: () => void;
   title: string;
   text: string;
+  textValues?: Record<string, string>;
   submitButtonText: string;
   onSubmit: () => void;
   submitButtonDataId?: string;
@@ -21,6 +22,7 @@ export const ConfirmationModal: React.FC<ConfirmationModalProps> = ({
   onClose,
   title,
   text,
+  textValues,
   onSubmit,
   submitButtonText,
   submitButtonDataId,
@@ -32,7 +34,7 @@ export const ConfirmationModal: React.FC<ConfirmationModalProps> = ({
   return (
     <Modal onClose={onClose} title={<FormattedMessage id={title} />} testId="confirmationModal">
       <div className={styles.content}>
-        <FormattedMessage id={text} />
+        <FormattedMessage id={text} values={textValues} />
         <div className={styles.buttonContent}>
           <Button
             className={styles.buttonWithMargin}

airbyte-webapp/src/components/connectorBuilder/Builder/AuthenticationSection.tsx

@@ -1,10 +1,10 @@
+import { inferredAuthValues } from "../types";
 import { BuilderCard } from "./BuilderCard";
 import { BuilderField } from "./BuilderField";
 import { BuilderFieldWithInputs } from "./BuilderFieldWithInputs";
 import { BuilderOneOf } from "./BuilderOneOf";
 import { BuilderOptional } from "./BuilderOptional";
 import { KeyValueListField } from "./KeyValueListField";
-import { UserInputField } from "./UserInputField";
 
 export const AuthenticationSection: React.FC = () => {
   return (
@@ -19,7 +19,7 @@ export const AuthenticationSection: React.FC = () => {
             label: "API Key",
             typeValue: "ApiKeyAuthenticator",
             default: {
-              api_token: "{{ config['api_key'] }}",
+              ...inferredAuthValues("ApiKeyAuthenticator"),
               header: "",
             },
             children: (
@@ -30,9 +30,12 @@ export const AuthenticationSection: React.FC = () => {
                   label="Header"
                   tooltip="HTTP header which should be set to the API Key"
                 />
-                <UserInputField
+                <BuilderField
+                  type="string"
+                  path="global.authenticator.api_token"
                   label="API Key"
                   tooltip="The API key issued by the service. Fill it in in the user inputs"
+                  readOnly
                 />
               </>
             ),
@@ -41,36 +44,48 @@ export const AuthenticationSection: React.FC = () => {
             label: "Bearer",
             typeValue: "BearerAuthenticator",
             default: {
-              api_token: "{{ config['api_key'] }}",
+              ...inferredAuthValues("BearerAuthenticator"),
             },
             children: (
-              <UserInputField
+              <BuilderField
+                type="string"
+                path="global.authenticator.api_token"
                 label="API Key"
                 tooltip="The API key issued by the service. Fill it in in the user inputs"
+                readOnly
               />
             ),
           },
           {
             label: "Basic HTTP",
             typeValue: "BasicHttpAuthenticator",
             default: {
-              username: "{{ config['username'] }}",
-              password: "{{ config['password'] }}",
+              ...inferredAuthValues("BasicHttpAuthenticator"),
             },
             children: (
               <>
-                <UserInputField label="Username" tooltip="The username for the login. Fill it in in the user inputs" />
-                <UserInputField label="Password" tooltip="The password for the login. Fill it in in the user inputs" />
+                <BuilderField
+                  type="string"
+                  path="global.authenticator.username"
+                  label="Username"
+                  tooltip="The username for the login. Fill it in in the user inputs"
+                  readOnly
+                />
+                <BuilderField
+                  type="string"
+                  path="global.authenticator.password"
+                  label="Password"
+                  tooltip="The password for the login. Fill it in in the user inputs"
+                  readOnly
+                />
               </>
             ),
           },
           {
             label: "OAuth",
             typeValue: "OAuthAuthenticator",
             default: {
-              client_id: "{{ config['client_id'] }}",
-              client_secret: "{{ config['client_secret'] }}",
-              refresh_token: "{{ config['client_refresh_token'] }}",
+              ...inferredAuthValues("OAuthAuthenticator"),
               refresh_request_body: [],
               token_refresh_endpoint: "",
             },
@@ -82,9 +97,27 @@ export const AuthenticationSection: React.FC = () => {
                   label="Token refresh endpoint"
                   tooltip="The URL to call to obtain a new access token"
                 />
-                <UserInputField label="Client ID" tooltip="The OAuth client ID" />
-                <UserInputField label="Client secret" tooltip="The OAuth client secret" />
-                <UserInputField label="Refresh token" tooltip="The OAuth refresh token" />
+                <BuilderField
+                  type="string"
+                  path="global.authenticator.client_id"
+                  label="Client ID"
+                  tooltip="The OAuth client ID. Fill it in in the user inputs"
+                  readOnly
+                />
+                <BuilderField
+                  type="string"
+                  path="global.authenticator.client_secret"
+                  label="Client secret"
+                  tooltip="The OAuth client secret. Fill it in in the user inputs"
+                  readOnly
+                />
+                <BuilderField
+                  type="string"
+                  path="global.authenticator.refresh_token"
+                  label="Refresh token"
+                  tooltip="The OAuth refresh token. Fill it in in the user inputs"
+                  readOnly
+                />
                 <BuilderOptional>
                   <BuilderField
                     type="array"
@@ -134,9 +167,7 @@ export const AuthenticationSection: React.FC = () => {
             label: "Session token",
             typeValue: "SessionTokenAuthenticator",
             default: {
-              username: "{{ config['username'] }}",
-              password: "{{ config['password'] }}",
-              session_token: "{{ config['session_token'] }}",
+              ...inferredAuthValues("SessionTokenAuthenticator"),
             },
             children: (
               <>
@@ -164,11 +195,26 @@ export const AuthenticationSection: React.FC = () => {
                   label="Validate session url"
                   tooltip="Url to validate passed session token"
                 />
-                <UserInputField label="Username" tooltip="The username" />
-                <UserInputField label="Password" tooltip="The password" />
-                <UserInputField
+                <BuilderField
+                  type="string"
+                  path="global.authenticator.username"
+                  label="Username"
+                  tooltip="The username for the login. Fill it in in the user inputs"
+                  readOnly
+                />
+                <BuilderField
+                  type="string"
+                  path="global.authenticator.password"
+                  label="Password"
+                  tooltip="The password for the login. Fill it in in the user inputs"
+                  readOnly
+                />
+                <BuilderField
+                  type="string"
+                  path="global.authenticator.session_token"
                   label="Session token"
-                  tooltip="Session token generated by user (if provided username and password are not required)"
+                  tooltip="Session token generated by user (if provided, username and password are not required). Fill it in in the user inputs"
+                  readOnly
                 />
               </>
             ),

airbyte-webapp/src/components/connectorBuilder/YamlEditor/YamlEditor.tsx

@@ -1,18 +1,23 @@
 import { useMonaco } from "@monaco-editor/react";
+import { useFormikContext } from "formik";
 import { load, YAMLException } from "js-yaml";
 import debounce from "lodash/debounce";
-import isMatch from "lodash/isMatch";
+import isEqual from "lodash/isEqual";
 import { editor } from "monaco-editor/esm/vs/editor/editor.api";
 import { useEffect, useMemo, useRef, useState } from "react";
 
 import { CodeEditor } from "components/ui/CodeEditor";
 
 import { ConnectorManifest } from "core/request/ConnectorManifest";
 import { useConfirmationModalService } from "hooks/services/ConfirmationModal";
-import { useConnectorBuilderFormState } from "services/connectorBuilder/ConnectorBuilderStateService";
+import {
+  useConnectorBuilderFormState,
+  useConnectorBuilderTestState,
+} from "services/connectorBuilder/ConnectorBuilderStateService";
 
 import { UiYamlToggleButton } from "../Builder/UiYamlToggleButton";
 import { DownloadYamlButton } from "../DownloadYamlButton";
+import { convertToBuilderFormValues } from "../manifestToBuilderForm";
 import { convertToManifest } from "../types";
 import styles from "./YamlEditor.module.scss";
 
@@ -21,6 +26,7 @@ interface YamlEditorProps {
 }
 
 export const YamlEditor: React.FC<YamlEditorProps> = ({ toggleYamlEditor }) => {
+  const { setValues } = useFormikContext();
   const { openConfirmationModal, closeConfirmationModal } = useConfirmationModalService();
   const yamlEditorRef = useRef<editor.IStandaloneCodeEditor>();
   const {
@@ -32,6 +38,7 @@ export const YamlEditor: React.FC<YamlEditorProps> = ({ toggleYamlEditor }) => {
     setYamlIsValid,
     setJsonManifest,
   } = useConnectorBuilderFormState();
+  const { streamListErrorMessage } = useConnectorBuilderTestState();
   const [yamlValue, setYamlValue] = useState(yamlManifest);
 
   // debounce the setJsonManifest calls so that it doesnt result in a network call for every keystroke
@@ -77,21 +84,27 @@ export const YamlEditor: React.FC<YamlEditorProps> = ({ toggleYamlEditor }) => {
   }, [yamlValue, monaco, debouncedSetJsonManifest, setYamlIsValid]);
 
   const yamlIsDirty = useMemo(() => {
-    return !isMatch(convertToManifest(builderFormValues), jsonManifest);
+    return !isEqual(convertToManifest(builderFormValues), jsonManifest);
   }, [jsonManifest, builderFormValues]);
 
   const handleToggleYamlEditor = () => {
     if (yamlIsDirty) {
-      openConfirmationModal({
-        text: "connectorBuilder.toggleModal.text",
-        title: "connectorBuilder.toggleModal.title",
-        submitButtonText: "connectorBuilder.toggleModal.submitButton",
-        onSubmit: () => {
-          setYamlIsValid(true);
-          toggleYamlEditor();
-          closeConfirmationModal();
-        },
-      });
+      try {
+        setValues(convertToBuilderFormValues(jsonManifest, builderFormValues, streamListErrorMessage));
+        toggleYamlEditor();
+      } catch (e) {
+        openConfirmationModal({
+          text: "connectorBuilder.toggleModal.text",
+          textValues: { error: e.message as string },
+          title: "connectorBuilder.toggleModal.title",
+          submitButtonText: "connectorBuilder.toggleModal.submitButton",
+          onSubmit: () => {
+            setYamlIsValid(true);
+            toggleYamlEditor();
+            closeConfirmationModal();
+          },
+        });
+      }
     } else {
       setYamlIsValid(true);
       toggleYamlEditor();