🪟🎉 Connector builder: Schema inferrer UI (#21154)

* fix stuff * add inferred schema to API * fix yaml changes * fix yaml formatting * add whitespace back * basic ui * advanced UI * Remove unused one * reset package lock * resolve merge conflicts * styling * show button and icon in the normal schema tab * restructure * handle yaml view * small fix * review comments * make monaco resize * review comments
airbytehq · Jan 13, 2023 · f6967f1 · f6967f1
1 parent ba7cbef
commit f6967f1
Show file tree

Hide file tree

Showing 17 changed files with 420 additions and 71 deletions.
diff --git a/airbyte-webapp/package-lock.json b/airbyte-webapp/package-lock.json
diff --git a/airbyte-webapp/package.json b/airbyte-webapp/package.json
@@ -39,10 +39,12 @@
     "@sentry/react": "^6.19.6",
     "@sentry/tracing": "^6.19.6",
     "@tanstack/react-table": "^8.7.0",
+    "@types/diff": "^5.0.2",
     "@types/segment-analytics": "^0.0.34",
     "@types/uuid": "^9.0.0",
     "classnames": "^2.3.1",
     "dayjs": "^1.11.3",
+    "diff": "^5.1.0",
     "firebase": "^9.8.2",
     "flat": "^5.0.2",
     "formik": "^2.2.9",

diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/StreamConfigView.module.scss b/airbyte-webapp/src/components/connectorBuilder/Builder/StreamConfigView.module.scss
@@ -52,13 +52,24 @@ $controlButtonWidth: 24px;
 }
 
 .errorMessage {
-  // hardcode height to prevent resizing when error message is hidden
-  height: 16px;
   color: colors.$red;
 }
 
 .schemaEditor {
-  height: 100%;
+  display: flex;
+  flex-grow: 1;
+  flex-direction: column;
+
+  // Needs to be set so the element is not overflowing its container due to the fixed Monaco height but respecting the flex flow.
+  // Monaco will remeasure its container to relayout itself correctly according to the changed height.
+  min-height: 0;
+}
+
+.editorContainer {
+  // Needs to be set so the element is not overflowing its container due to the fixed Monaco height but respecting the flex flow.
+  // Monaco will remeasure its container to relayout itself correctly according to the changed height.
+  min-height: 0;
+  flex: 1 1 0;
 }
 
 .multiStreams {

diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/StreamConfigView.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/StreamConfigView.tsx
@@ -2,18 +2,25 @@ import { faTrashCan, faCopy } from "@fortawesome/free-regular-svg-icons";
 import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
 import classNames from "classnames";
 import { useField } from "formik";
-import { useState } from "react";
+import { useMemo, useState } from "react";
 import React from "react";
 import { FormattedMessage, useIntl } from "react-intl";
 
 import Indicator from "components/Indicator";
+import { Button } from "components/ui/Button";
 import { CodeEditor } from "components/ui/CodeEditor";
 import { Text } from "components/ui/Text";
 
 import { useConfirmationModalService } from "hooks/services/ConfirmationModal";
-import { BuilderView, useConnectorBuilderFormState } from "services/connectorBuilder/ConnectorBuilderStateService";
+import {
+  BuilderView,
+  useConnectorBuilderFormState,
+  useConnectorBuilderTestState,
+} from "services/connectorBuilder/ConnectorBuilderStateService";
 
+import { SchemaConflictIndicator } from "../SchemaConflictIndicator";
 import { BuilderStream } from "../types";
+import { formatJson } from "../utils";
 import { AddStreamButton } from "./AddStreamButton";
 import { BuilderCard } from "./BuilderCard";
 import { BuilderConfigView } from "./BuilderConfigView";
@@ -124,7 +131,12 @@ const StreamControls = ({
   const [field, , helpers] = useField<BuilderStream[]>("streams");
   const { openConfirmationModal, closeConfirmationModal } = useConfirmationModalService();
   const { setSelectedView } = useConnectorBuilderFormState();
-  const [, meta] = useField<string | undefined>(streamFieldPath("schema"));
+  const { streamRead: readStream } = useConnectorBuilderTestState();
+  const [schema, meta] = useField<string | undefined>(streamFieldPath("schema"));
+  const formattedDetectedSchema = useMemo(
+    () => readStream.data?.inferred_schema && formatJson(readStream.data?.inferred_schema, true),
+    [readStream.data?.inferred_schema]
+  );
   const hasSchemaErrors = Boolean(meta.error);
 
   const handleDelete = () => {
@@ -154,6 +166,7 @@ const StreamControls = ({
         selected={selectedTab === "schema"}
         onSelect={() => setSelectedTab("schema")}
         showErrorIndicator={hasSchemaErrors}
+        showSchemaConflictIndicator={Boolean(formattedDetectedSchema && schema.value !== formattedDetectedSchema)}
       />
       <AddStreamButton
         onAddStream={(addedStreamNum) => {
@@ -178,32 +191,55 @@ const StreamTab = ({
   label,
   onSelect,
   showErrorIndicator,
+  showSchemaConflictIndicator,
 }: {
   selected: boolean;
   label: string;
   onSelect: () => void;
   showErrorIndicator?: boolean;
+  showSchemaConflictIndicator?: boolean;
 }) => (
   <button type="button" className={classNames(styles.tab, { [styles.selectedTab]: selected })} onClick={onSelect}>
     {label}
     {showErrorIndicator && <Indicator />}
+    {showSchemaConflictIndicator && <SchemaConflictIndicator />}
   </button>
 );
 
 const SchemaEditor = ({ streamFieldPath }: { streamFieldPath: (fieldPath: string) => string }) => {
   const [field, meta, helpers] = useField<string | undefined>(streamFieldPath("schema"));
+  const { streamRead } = useConnectorBuilderTestState();
+
+  const showImportButton = !field.value && streamRead.data?.inferred_schema;
 
   return (
     <>
-      <CodeEditor
-        value={field.value || ""}
-        language="json"
-        theme="airbyte-light"
-        onChange={(val: string | undefined) => {
-          helpers.setValue(val);
-        }}
-      />
-      <Text className={styles.errorMessage}>{meta.error && <FormattedMessage id={meta.error} />}</Text>
+      {showImportButton && (
+        <Button
+          full
+          variant="secondary"
+          onClick={() => {
+            helpers.setValue(formatJson(streamRead.data?.inferred_schema, true));
+          }}
+        >
+          <FormattedMessage id="connectorBuilder.useSchemaButton" />
+        </Button>
+      )}
+      <div className={styles.editorContainer}>
+        <CodeEditor
+          value={field.value || ""}
+          language="json"
+          theme="airbyte-light"
+          onChange={(val: string | undefined) => {
+            helpers.setValue(val);
+          }}
+        />
+      </div>
+      {meta.error && (
+        <Text className={styles.errorMessage}>
+          <FormattedMessage id={meta.error} />
+        </Text>
+      )}
     </>
   );
 };
diff --git a/airbyte-webapp/src/components/connectorBuilder/SchemaConflictIndicator.module.scss b/airbyte-webapp/src/components/connectorBuilder/SchemaConflictIndicator.module.scss
@@ -0,0 +1,5 @@
+@use "scss/colors";
+
+.schemaConflictIcon {
+  color: colors.$yellow-400;
+}
diff --git a/airbyte-webapp/src/components/connectorBuilder/SchemaConflictIndicator.tsx b/airbyte-webapp/src/components/connectorBuilder/SchemaConflictIndicator.tsx
@@ -0,0 +1,13 @@
+import { faWarning } from "@fortawesome/free-solid-svg-icons";
+import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
+import { FormattedMessage } from "react-intl";
+
+import { Tooltip } from "components/ui/Tooltip";
+
+import styles from "./SchemaConflictIndicator.module.scss";
+
+export const SchemaConflictIndicator: React.FC = () => (
+  <Tooltip control={<FontAwesomeIcon icon={faWarning} className={styles.schemaConflictIcon} />}>
+    <FormattedMessage id="connectorBuilder.differentSchemaDescription" />
+  </Tooltip>
+);
diff --git a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/LogsDisplay.tsx b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/LogsDisplay.tsx
@@ -6,8 +6,8 @@ import { Text } from "components/ui/Text";
 
 import { StreamReadLogsItem } from "core/request/ConnectorBuilderClient";
 
+import { formatJson } from "../utils";
 import styles from "./LogsDisplay.module.scss";
-import { formatJson } from "./utils";
 
 interface LogsDisplayProps {
   logs: StreamReadLogsItem[];

diff --git a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/PageDisplay.module.scss b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/PageDisplay.module.scss
@@ -4,19 +4,21 @@
 .tabList {
   flex: 0 0 auto;
   display: flex;
+  overflow-y: auto;
 }
 
 .tab {
-  flex: 1;
+  flex-grow: 1;
+  flex-shrink: 0;
   background-color: transparent;
   border: 0;
+  white-space: nowrap;
 }
 
 .tabTitle {
   border-bottom: variables.$border-thin solid colors.$grey-50;
   color: colors.$grey-300;
   font-weight: 500;
-  font-size: 10px;
   cursor: pointer;
   padding: 3px;
 }

diff --git a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/PageDisplay.tsx b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/PageDisplay.tsx
@@ -1,17 +1,26 @@
 import { Tab } from "@headlessui/react";
 import classNames from "classnames";
-import { useMemo } from "react";
+import { useField } from "formik";
+import React, { useMemo } from "react";
 import { useIntl } from "react-intl";
 
+import { FlexContainer } from "components/ui/Flex";
 import { Text } from "components/ui/Text";
 
-import { StreamReadSlicesItemPagesItem } from "core/request/ConnectorBuilderClient";
+import { StreamReadInferredSchema, StreamReadSlicesItemPagesItem } from "core/request/ConnectorBuilderClient";
+import {
+  useConnectorBuilderFormState,
+  useConnectorBuilderTestState,
+} from "services/connectorBuilder/ConnectorBuilderStateService";
 
+import { SchemaConflictIndicator } from "../SchemaConflictIndicator";
+import { formatJson } from "../utils";
 import styles from "./PageDisplay.module.scss";
-import { formatJson } from "./utils";
+import { SchemaDiffView } from "./SchemaDiffView";
 
 interface PageDisplayProps {
   page: StreamReadSlicesItemPagesItem;
+  inferredSchema?: StreamReadInferredSchema;
   className?: string;
 }
 
@@ -21,12 +30,17 @@ interface TabData {
   content: string;
 }
 
-export const PageDisplay: React.FC<PageDisplayProps> = ({ page, className }) => {
+export const PageDisplay: React.FC<PageDisplayProps> = ({ page, className, inferredSchema }) => {
   const { formatMessage } = useIntl();
 
+  const { editorView } = useConnectorBuilderFormState();
+  const { testStreamIndex } = useConnectorBuilderTestState();
+  const [field] = useField(`streams[${testStreamIndex}].schema`);
+
   const formattedRecords = useMemo(() => formatJson(page.records), [page.records]);
   const formattedRequest = useMemo(() => formatJson(page.request), [page.request]);
   const formattedResponse = useMemo(() => formatJson(page.response), [page.response]);
+  const formattedSchema = useMemo(() => inferredSchema && formatJson(inferredSchema, true), [inferredSchema]);
 
   let defaultTabIndex = 0;
   const tabs: TabData[] = [
@@ -58,22 +72,41 @@ export const PageDisplay: React.FC<PageDisplayProps> = ({ page, className }) =>
   return (
     <div className={classNames(className)}>
       <Tab.Group defaultIndex={defaultTabIndex}>
-        <Tab.List className={styles.tabList}>
-          {tabs.map((tab) => (
-            <Tab className={styles.tab} key={tab.key}>
-              {({ selected }) => (
-                <Text className={classNames(styles.tabTitle, { [styles.selected]: selected })}>{tab.title}</Text>
-              )}
-            </Tab>
-          ))}
-        </Tab.List>
-        <Tab.Panels className={styles.tabPanelContainer}>
-          {tabs.map((tab) => (
-            <Tab.Panel className={styles.tabPanel} key={tab.key}>
-              <pre>{tab.content}</pre>
-            </Tab.Panel>
-          ))}
-        </Tab.Panels>
+        <FlexContainer direction="column">
+          <Tab.List className={styles.tabList}>
+            {tabs.map((tab) => (
+              <Tab className={styles.tab} key={tab.key}>
+                {({ selected }) => (
+                  <Text className={classNames(styles.tabTitle, { [styles.selected]: selected })}>{tab.title}</Text>
+                )}
+              </Tab>
+            ))}
+            {inferredSchema && (
+              <Tab className={styles.tab}>
+                {({ selected }) => (
+                  <Text className={classNames(styles.tabTitle, { [styles.selected]: selected })} as="div">
+                    <FlexContainer direction="row" justifyContent="center">
+                      {formatMessage({ id: "connectorBuilder.schemaTab" })}
+                      {editorView === "ui" && field.value !== formattedSchema && <SchemaConflictIndicator />}
+                    </FlexContainer>
+                  </Text>
+                )}
+              </Tab>
+            )}
+          </Tab.List>
+          <Tab.Panels className={styles.tabPanelContainer}>
+            {tabs.map((tab) => (
+              <Tab.Panel className={styles.tabPanel} key={tab.key}>
+                <pre>{tab.content}</pre>
+              </Tab.Panel>
+            ))}
+            {inferredSchema && (
+              <Tab.Panel className={styles.tabPanel}>
+                <SchemaDiffView inferredSchema={inferredSchema} />
+              </Tab.Panel>
+            )}
+          </Tab.Panels>
+        </FlexContainer>
       </Tab.Group>
     </div>
   );