Ingestion: Added form in Big Query type to edit the queries. (#5431)

datahub-web-react/src/app/ingest/source/builder/RecipeForm/FormField.tsx

@@ -1,5 +1,5 @@
-import { Button, Checkbox, Form, Input, Tooltip } from 'antd';
 import React from 'react';
+import { Button, Checkbox, Form, Input, Select, Tooltip } from 'antd';
 import styled from 'styled-components/macro';
 import { MinusCircleOutlined, PlusOutlined, QuestionCircleOutlined } from '@ant-design/icons';
 import { FieldType, RecipeField } from './utils';
@@ -25,6 +25,10 @@ const StyledRemoveIcon = styled(MinusCircleOutlined)`
     margin-left: 10px;
 `;
 
+const StyledSelectField = styled(Select)`
+    margin-left: 10px;
+`;
+
 const StyledFormItem = styled(Form.Item)<{ alignLeft: boolean; removeMargin: boolean }>`
     margin-bottom: ${(props) => (props.removeMargin ? '0' : '16px')};
 
@@ -52,6 +56,10 @@ interface ListFieldProps {
     removeMargin?: boolean;
 }
 
+interface SelectFieldProps {
+    field: RecipeField;
+}
+
 function ListField({ field, removeMargin }: ListFieldProps) {
     return (
         <Form.List name={field.name}>
@@ -80,6 +88,25 @@ function ListField({ field, removeMargin }: ListFieldProps) {
     );
 }
 
+function SelectField({ field }: SelectFieldProps) {
+    return (
+        <Form.Item
+            name={field.name}
+            label={field.label}
+            tooltip={field.tooltip}
+            style={{ flexDirection: 'row', width: '80%', display: 'flex', alignItems: 'baseline' }}
+        >
+            {field.options && (
+                <StyledSelectField>
+                    {field.options.map((option) => (
+                        <Select.Option value={option.value}>{option.label}</Select.Option>
+                    ))}
+                </StyledSelectField>
+            )}
+        </Form.Item>
+    );
+}
+
 interface Props {
     field: RecipeField;
     removeMargin?: boolean;
@@ -90,6 +117,8 @@ function FormField(props: Props) {
 
     if (field.type === FieldType.LIST) return <ListField field={field} removeMargin={removeMargin} />;
 
+    if (field.type === FieldType.SELECT) return <SelectField field={field} />;
+
     const isBoolean = field.type === FieldType.BOOLEAN;
     const input = isBoolean ? <Checkbox /> : <Input />;
     const valuePropName = isBoolean ? 'checked' : 'value';

datahub-web-react/src/app/ingest/source/builder/RecipeForm/utils.ts → datahub-web-react/src/app/ingest/source/builder/RecipeForm/utils.tsx

@@ -1,20 +1,30 @@
+import React from 'react';
 import { set, get } from 'lodash';
 import { SNOWFLAKE } from '../../conf/snowflake/snowflake';
+import { BIGQUERY } from '../../conf/bigquery/bigquery';
+import { REDSHIFT } from '../../conf/redshift/redshift';
 
 export enum FieldType {
     TEXT,
     BOOLEAN,
     LIST,
+    SELECT,
+}
+
+interface Option {
+    label: string;
+    value: string;
 }
 
 export interface RecipeField {
     name: string;
     label: string;
-    tooltip: string;
+    tooltip: string | React.ReactNode;
     type: FieldType;
     fieldPath: string;
     rules: any[] | null;
     section?: string;
+    options?: Option[];
     getValueFromRecipeOverride?: (recipe: any) => any;
     setValueOnRecipeOverride?: (recipe: any, value: any) => any;
 }
@@ -102,6 +112,96 @@ export const SNOWFLAKE_ROLE: RecipeField = {
     rules: null,
 };
 
+export const BIGQUERY_PROJECT_ID: RecipeField = {
+    name: 'project_id',
+    label: 'BigQuery Project ID',
+    tooltip: 'Project ID where you have rights to run queries and create tables.',
+    type: FieldType.TEXT,
+    fieldPath: 'source.config.project_id',
+    rules: null,
+};
+
+export const BIGQUERY_CREDENTIAL_PROJECT_ID: RecipeField = {
+    name: 'credential.project_id',
+    label: 'Credentials Project ID',
+    tooltip: 'Project id to set the credentials.',
+    type: FieldType.TEXT,
+    fieldPath: 'source.config.credential.project_id',
+    rules: null,
+};
+
+export const BIGQUERY_PRIVATE_KEY_ID: RecipeField = {
+    name: 'credential.private_key_id',
+    label: 'Private Key Id',
+    tooltip: 'Private key id.',
+    type: FieldType.TEXT,
+    fieldPath: 'source.config.credential.private_key_id',
+    rules: null,
+};
+
+export const BIGQUERY_PRIVATE_KEY: RecipeField = {
+    name: 'credential.private_key',
+    label: 'Private Key',
+    tooltip: 'Private key in a form of "-----BEGIN PRIVATE KEY-----\nprivate-key\n-----END PRIVATE KEY-----\n".',
+    type: FieldType.TEXT,
+    fieldPath: 'source.config.credential.private_key',
+    rules: null,
+};
+
+export const BIGQUERY_CLIENT_EMAIL: RecipeField = {
+    name: 'credential.client_email',
+    label: 'Client Email',
+    tooltip: 'Client email.',
+    type: FieldType.TEXT,
+    fieldPath: 'source.config.credential.client_email',
+    rules: null,
+};
+
+export const BIGQUERY_CLIENT_ID: RecipeField = {
+    name: 'credential.client_id',
+    label: 'Client ID',
+    tooltip: 'Client ID.',
+    type: FieldType.TEXT,
+    fieldPath: 'source.config.credential.client_id',
+    rules: null,
+};
+
+export const REDSHIFT_HOST_PORT: RecipeField = {
+    name: 'host_port',
+    label: 'Host Port',
+    tooltip: 'Host URL.',
+    type: FieldType.TEXT,
+    fieldPath: 'source.config.host_port',
+    rules: null,
+};
+
+export const REDSHIFT_DATABASE: RecipeField = {
+    name: 'database',
+    label: 'Database',
+    tooltip: 'Database (catalog).',
+    type: FieldType.TEXT,
+    fieldPath: 'source.config.database',
+    rules: null,
+};
+
+export const REDSHIFT_USERNAME: RecipeField = {
+    name: 'redshift.username',
+    label: 'Redshift username',
+    tooltip: 'Username',
+    type: FieldType.TEXT,
+    fieldPath: 'source.config.username',
+    rules: null,
+};
+
+export const REDSHIFT_PASSWORD: RecipeField = {
+    name: 'redshift.password',
+    label: 'Redshift password',
+    tooltip: 'Password',
+    type: FieldType.TEXT,
+    fieldPath: 'source.config.password',
+    rules: null,
+};
+
 const includeLineageFieldPathA = 'source.config.include_table_lineage';
 const includeLineageFieldPathB = 'source.config.include_view_lineage';
 export const INCLUDE_LINEAGE: RecipeField = {
@@ -157,6 +257,47 @@ export const STATEFUL_INGESTION_ENABLED: RecipeField = {
     rules: null,
 };
 
+export const UPSTREAM_LINEAGE_IN_REPORT: RecipeField = {
+    name: 'upstream_lineage_in_report',
+    label: 'Include Upstream Lineage In Report.',
+    tooltip: 'Useful for debugging lineage information. Set to True to see the raw lineage created internally.',
+    type: FieldType.BOOLEAN,
+    fieldPath: 'source.config.upstream_lineage_in_report',
+    rules: null,
+};
+
+const TableLineageModeTooltip = () => {
+    return (
+        <div>
+            <p>
+                Which table lineage collector mode to use. Check out{' '}
+                <a
+                    href="https://datahubproject.io/docs/generated/ingestion/sources/redshift/#config-details"
+                    target="_blank"
+                    rel="noreferrer"
+                >
+                    the documentation
+                </a>{' '}
+                explaining the difference between the three available modes.
+            </p>
+        </div>
+    );
+};
+
+export const TABLE_LINEAGE_MODE: RecipeField = {
+    name: 'table_lineage_mode',
+    label: 'Table Lineage Mode',
+    tooltip: TableLineageModeTooltip,
+    type: FieldType.SELECT,
+    fieldPath: 'source.config.table_lineage_mode',
+    rules: null,
+    options: [
+        { label: 'stl_scan_based', value: 'stl_scan_based' },
+        { label: 'sql_based', value: 'sql_based' },
+        { label: 'mixed', value: 'mixed' },
+    ],
+};
+
 const databaseAllowFieldPath = 'source.config.database_pattern.allow';
 export const DATABASE_ALLOW: RecipeField = {
     name: 'database_pattern.allow',
@@ -282,6 +423,23 @@ export const RECIPE_FIELDS = {
             VIEW_DENY,
         ],
     },
+    [BIGQUERY]: {
+        fields: [
+            BIGQUERY_PROJECT_ID,
+            BIGQUERY_CREDENTIAL_PROJECT_ID,
+            BIGQUERY_PRIVATE_KEY,
+            BIGQUERY_PRIVATE_KEY_ID,
+            BIGQUERY_CLIENT_EMAIL,
+            BIGQUERY_CLIENT_ID,
+        ],
+        advancedFields: [INCLUDE_LINEAGE, PROFILING_ENABLED, STATEFUL_INGESTION_ENABLED, UPSTREAM_LINEAGE_IN_REPORT],
+        filterFields: [TABLE_ALLOW, TABLE_DENY, SCHEMA_ALLOW, SCHEMA_DENY, VIEW_ALLOW, VIEW_DENY],
+    },
+    [REDSHIFT]: {
+        fields: [REDSHIFT_HOST_PORT, REDSHIFT_DATABASE, REDSHIFT_USERNAME, REDSHIFT_PASSWORD],
+        advancedFields: [INCLUDE_LINEAGE, PROFILING_ENABLED, STATEFUL_INGESTION_ENABLED, TABLE_LINEAGE_MODE],
+        filterFields: [TABLE_ALLOW, TABLE_DENY, SCHEMA_ALLOW, SCHEMA_DENY, VIEW_ALLOW, VIEW_DENY],
+    },
 };
 
 export const CONNECTORS_WITH_FORM = new Set(Object.keys(RECIPE_FIELDS));

datahub-web-react/src/app/ingest/source/conf/bigquery/bigquery.ts

@@ -21,8 +21,10 @@ source:
             client_id: # Your BQ client id, e.g. "123456678890"
 `;
 
+export const BIGQUERY = 'bigquery';
+
 const bigqueryConfig: SourceConfig = {
-    type: 'bigquery',
+    type: BIGQUERY,
     placeholderRecipe,
     displayName: 'BigQuery',
     docsUrl: 'https://datahubproject.io/docs/generated/ingestion/sources/bigquery/',

datahub-web-react/src/app/ingest/source/conf/redshift/redshift.ts

@@ -23,8 +23,10 @@ source:
             enabled: false
 `;
 
+export const REDSHIFT = 'redshift';
+
 const redshiftConfig: SourceConfig = {
-    type: 'redshift',
+    type: REDSHIFT,
     placeholderRecipe,
     displayName: 'Redshift',
     docsUrl: 'https://datahubproject.io/docs/generated/ingestion/sources/redshift/',