Merge branch 'master' into stateful_fix_percentage_cal

mayurinehate · Oct 11, 2024 · 9d1cf2b · 9d1cf2b
2 parents c2a93e8 + 43c185d
commit 9d1cf2b
Show file tree

Hide file tree

Showing 22 changed files with 699 additions and 132 deletions.
diff --git a/.github/workflows/airflow-plugin.yml b/.github/workflows/airflow-plugin.yml
@@ -54,6 +54,9 @@ jobs:
           - python-version: "3.11"
             extra_pip_requirements: "apache-airflow~=2.9.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.9.3/constraints-3.11.txt"
             extra_pip_extras: plugin-v2
+          - python-version: "3.11"
+            extra_pip_requirements: "apache-airflow~=2.10.2 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.10.2/constraints-3.11.txt"
+            extra_pip_extras: plugin-v2
       fail-fast: false
     steps:
       - name: Set up JDK 17

diff --git a/.github/workflows/docker-unified.yml b/.github/workflows/docker-unified.yml
@@ -186,9 +186,11 @@ jobs:
         with:
           image: ${{ env.DATAHUB_GMS_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
       - name: Run Trivy vulnerability scanner
-        uses: aquasecurity/trivy-action@0.25.0
+        uses: aquasecurity/trivy-action@0.26.0
         env:
           TRIVY_OFFLINE_SCAN: true
+          TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db:2,ghcr.io/aquasecurity/trivy-db:2
+          TRIVY_JAVA_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-java-db:1,ghcr.io/aquasecurity/trivy-java-db:1
         with:
           image-ref: ${{ env.DATAHUB_GMS_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
           format: "template"
@@ -250,9 +252,11 @@ jobs:
         with:
           image: ${{ env.DATAHUB_MAE_CONSUMER_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
       - name: Run Trivy vulnerability scanner
-        uses: aquasecurity/trivy-action@0.25.0
+        uses: aquasecurity/trivy-action@0.26.0
         env:
           TRIVY_OFFLINE_SCAN: true
+          TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db:2,ghcr.io/aquasecurity/trivy-db:2
+          TRIVY_JAVA_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-java-db:1,ghcr.io/aquasecurity/trivy-java-db:1
         with:
           image-ref: ${{ env.DATAHUB_MAE_CONSUMER_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
           format: "template"
@@ -314,9 +318,11 @@ jobs:
         with:
           image: ${{ env.DATAHUB_MCE_CONSUMER_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
       - name: Run Trivy vulnerability scanner
-        uses: aquasecurity/trivy-action@0.25.0
+        uses: aquasecurity/trivy-action@0.26.0
         env:
           TRIVY_OFFLINE_SCAN: true
+          TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db:2,ghcr.io/aquasecurity/trivy-db:2
+          TRIVY_JAVA_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-java-db:1,ghcr.io/aquasecurity/trivy-java-db:1
         with:
           image-ref: ${{ env.DATAHUB_MCE_CONSUMER_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
           format: "template"
@@ -378,9 +384,11 @@ jobs:
         with:
           image: ${{ env.DATAHUB_UPGRADE_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
       - name: Run Trivy vulnerability scanner
-        uses: aquasecurity/trivy-action@0.25.0
+        uses: aquasecurity/trivy-action@0.26.0
         env:
           TRIVY_OFFLINE_SCAN: true
+          TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db:2,ghcr.io/aquasecurity/trivy-db:2
+          TRIVY_JAVA_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-java-db:1,ghcr.io/aquasecurity/trivy-java-db:1
         with:
           image-ref: ${{ env.DATAHUB_UPGRADE_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
           format: "template"
@@ -444,9 +452,11 @@ jobs:
         with:
           image: ${{ env.DATAHUB_FRONTEND_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
       - name: Run Trivy vulnerability scanner
-        uses: aquasecurity/trivy-action@0.25.0
+        uses: aquasecurity/trivy-action@0.26.0
         env:
           TRIVY_OFFLINE_SCAN: true
+          TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db:2,ghcr.io/aquasecurity/trivy-db:2
+          TRIVY_JAVA_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-java-db:1,ghcr.io/aquasecurity/trivy-java-db:1
         with:
           image-ref: ${{ env.DATAHUB_FRONTEND_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
           format: "template"
@@ -498,9 +508,11 @@ jobs:
         with:
           image: ${{ env.DATAHUB_KAFKA_SETUP_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
       - name: Run Trivy vulnerability scanner
-        uses: aquasecurity/trivy-action@0.25.0
+        uses: aquasecurity/trivy-action@0.26.0
         env:
           TRIVY_OFFLINE_SCAN: true
+          TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db:2,ghcr.io/aquasecurity/trivy-db:2
+          TRIVY_JAVA_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-java-db:1,ghcr.io/aquasecurity/trivy-java-db:1
         with:
           image-ref: ${{ env.DATAHUB_KAFKA_SETUP_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
           format: "template"
@@ -552,9 +564,11 @@ jobs:
         with:
           image: ${{ env.DATAHUB_MYSQL_SETUP_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
       - name: Run Trivy vulnerability scanner
-        uses: aquasecurity/trivy-action@0.25.0
+        uses: aquasecurity/trivy-action@0.26.0
         env:
           TRIVY_OFFLINE_SCAN: true
+          TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db:2,ghcr.io/aquasecurity/trivy-db:2
+          TRIVY_JAVA_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-java-db:1,ghcr.io/aquasecurity/trivy-java-db:1
         with:
           image-ref: ${{ env.DATAHUB_MYSQL_SETUP_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
           format: "template"
@@ -606,9 +620,11 @@ jobs:
         with:
           image: ${{ env.DATAHUB_ELASTIC_SETUP_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
       - name: Run Trivy vulnerability scanner
-        uses: aquasecurity/trivy-action@0.25.0
+        uses: aquasecurity/trivy-action@0.26.0
         env:
           TRIVY_OFFLINE_SCAN: true
+          TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db:2,ghcr.io/aquasecurity/trivy-db:2
+          TRIVY_JAVA_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-java-db:1,ghcr.io/aquasecurity/trivy-java-db:1
         with:
           image-ref: ${{ env.DATAHUB_ELASTIC_SETUP_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
           format: "template"
@@ -808,9 +824,11 @@ jobs:
         with:
           image: ${{ env.DATAHUB_INGESTION_IMAGE }}:${{ needs.datahub_ingestion_slim_build.outputs.tag }}
       - name: Run Trivy vulnerability scanner Slim Image
-        uses: aquasecurity/trivy-action@0.25.0
+        uses: aquasecurity/trivy-action@0.26.0
         env:
           TRIVY_OFFLINE_SCAN: true
+          TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db:2,ghcr.io/aquasecurity/trivy-db:2
+          TRIVY_JAVA_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-java-db:1,ghcr.io/aquasecurity/trivy-java-db:1
         with:
           image-ref: ${{ env.DATAHUB_INGESTION_IMAGE }}:${{ needs.datahub_ingestion_slim_build.outputs.tag }}
           format: "template"
@@ -896,9 +914,11 @@ jobs:
         with:
           image: ${{ env.DATAHUB_INGESTION_IMAGE }}:${{ needs.datahub_ingestion_full_build.outputs.tag }}
       - name: Run Trivy vulnerability scanner Full Image
-        uses: aquasecurity/trivy-action@0.25.0
+        uses: aquasecurity/trivy-action@0.26.0
         env:
           TRIVY_OFFLINE_SCAN: true
+          TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db:2,ghcr.io/aquasecurity/trivy-db:2
+          TRIVY_JAVA_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-java-db:1,ghcr.io/aquasecurity/trivy-java-db:1
         with:
           image-ref: ${{ env.DATAHUB_INGESTION_IMAGE }}:${{ needs.datahub_ingestion_full_build.outputs.tag }}
           format: "template"

diff --git a/build.gradle b/build.gradle
@@ -107,8 +107,8 @@ project.ext.externalDependency = [
     'antlr4Runtime': 'org.antlr:antlr4-runtime:4.9.3',
     'antlr4': 'org.antlr:antlr4:4.9.3',
     'assertJ': 'org.assertj:assertj-core:3.11.1',
-    'avro': 'org.apache.avro:avro:1.11.3',
-    'avroCompiler': 'org.apache.avro:avro-compiler:1.11.3',
+    'avro': 'org.apache.avro:avro:1.11.4',
+    'avroCompiler': 'org.apache.avro:avro-compiler:1.11.4',
     'awsGlueSchemaRegistrySerde': 'software.amazon.glue:schema-registry-serde:1.1.17',
     'awsMskIamAuth': 'software.amazon.msk:aws-msk-iam-auth:2.0.3',
     'awsS3': 'software.amazon.awssdk:s3:2.26.21',

diff --git a/datahub-web-react/src/app/entity/shared/containers/profile/EntityProfile.tsx b/datahub-web-react/src/app/entity/shared/containers/profile/EntityProfile.tsx
@@ -323,7 +323,13 @@ export const EntityProfile = <T, U>({
                 {showBrowseBar && <EntityProfileNavBar urn={urn} entityType={entityType} />}
                 {entityData?.status?.removed === true && (
                     <Alert
-                        message="This entity is not discoverable via search or lineage graph. Contact your DataHub admin for more information."
+                        message={
+                            <>
+                                This entity is marked as soft-deleted, likely due to stateful ingestion or a manual
+                                deletion command, and will not appear in search or lineage graphs. Contact your DataHub
+                                admin for more information.
+                            </>
+                        }
                         banner
                     />
                 )}

diff --git a/docker/build.gradle b/docker/build.gradle
@@ -31,6 +31,8 @@ ext {
     ]
     // Postgres
     pg_quickstart_modules = quickstart_modules - [':docker:mysql-setup'] + [':docker:postgres-setup']
+
+    revision = 1 // increment to trigger rebuild
 }
 
 tasks.register('minDockerCompose2.20', Exec) {

diff --git a/docs-website/src/pages/cloud/DemoForm/styles.module.scss b/docs-website/src/pages/cloud/DemoForm/styles.module.scss
@@ -46,9 +46,7 @@
   .bookButton {
     display: block;
   }
-  .formContainer {
-    display: none;
-  }
+
   .productTourButton {
     text-align: center!important;
   }

diff --git a/docs-website/src/pages/cloud/DemoFormModal/index.jsx b/docs-website/src/pages/cloud/DemoFormModal/index.jsx
@@ -0,0 +1,17 @@
+import React, { useEffect } from 'react';
+import styles from "./styles.module.scss";
+import DemoForm from '../DemoForm';
+
+const DemoFormModal = ({ formId, handleCloseModal }) => {
+
+  return (
+    <div className={styles.modal}>
+    <div className={styles.modalContent}>
+      <button className={styles.closeButton} onClick={handleCloseModal}>✕</button>
+      <DemoForm formId={formId} />
+    </div>
+  </div>
+  );
+};
+
+export default DemoFormModal;
diff --git a/docs-website/src/pages/cloud/DemoFormModal/styles.module.scss b/docs-website/src/pages/cloud/DemoFormModal/styles.module.scss
@@ -0,0 +1,45 @@
+
+.modal {
+  position: fixed;
+  top: 0;
+  left: 0;
+  width: 100%;
+  height: 100%;
+  background: rgba(0, 0, 0, 0.5); /* Modal background */
+  display: flex;
+  justify-content: center;
+  align-items: center;
+  z-index: 1000;
+  overflow-y: scroll;
+
+  .modalContent {
+    width: 90%;
+    max-width: 540px;
+    background: #fff;
+    border-radius: 16px;
+    position: relative;
+    box-shadow: 0px 0px 20px rgba(0, 0, 0, 0.1);
+    animation: modalShow 0.3s ease-in-out;
+
+    .closeButton {
+      position: absolute;
+      top: 1rem;
+      right: 1rem;
+      background: none;
+      border: none;
+      font-size: 1.5rem;
+      cursor: pointer;
+    }
+  }
+}
+
+@keyframes modalShow {
+  from {
+    opacity: 0;
+    transform: translateY(-20px);
+  }
+  to {
+    opacity: 1;
+    transform: translateY(0);
+  }
+}
diff --git a/docs-website/src/pages/cloud/Hero/index.js b/docs-website/src/pages/cloud/Hero/index.js
@@ -1,38 +1,56 @@
-import React, { useEffect } from 'react';
+import React, { useState } from 'react';
 import clsx from "clsx";
 import Link from "@docusaurus/Link";
 import styles from "./styles.module.scss";
 import ScrollingCustomers from '../CompanyLogos';
 import DemoForm from '../DemoForm';
+import DemoFormModal from '../DemoFormModal';
 
 const Hero = () => {
+  const [isModalOpen, setIsModalOpen] = useState(false);
+
+  const handleOpenModal = () => setIsModalOpen(true);
+  const handleCloseModal = () => setIsModalOpen(false);
+
   return (
     <header className={clsx("hero", styles.hero)}>
       <div className="container">
         <div className="hero__content">
           <div className="row row__padded">
             <div className={clsx(styles.hero__cta, styles.col, "col col--7")}>
-                <h1 className={clsx("hero__title", styles.hero__title)}>DataHub Cloud</h1>
-                <div className={clsx("hero__subtitle", styles.hero__subtitle)}>
+              <h1 className={clsx("hero__title", styles.hero__title)}>DataHub Cloud</h1>
+              <div className={clsx("hero__subtitle", styles.hero__subtitle)}>
                 Experience the premium version of DataHub
-                  <div style={{ fontWeight: "500" }}>
-                    with Observability and Governance built-in.
-                  </div>
+                <div style={{ fontWeight: "500" }}>
+                  with Observability and Governance built-in.
                 </div>
-                <Link className={clsx(styles.button, styles.bookButton, "button button--primary button--lg")} to="https://www.acryldata.io/datahub-sign-up?utm_source=datahub&utm_medium=referral&utm_campaign=acryl_signup">
-                  Book Demo
-                </Link>
-                <Link className={clsx(styles.button, styles.productTourButton, "button button--secondary button--lg")} to="https://www.acryldata.io/tour">
-                  Live Product Tour →
-                </Link>
-                <ScrollingCustomers />
+              </div>
+
+              <button
+                className={clsx(styles.button, styles.bookButton, "button button--primary button--lg")}
+                onClick={handleOpenModal}
+              >
+                Book Demo
+              </button>
+
+              <Link
+                className={clsx(styles.button, styles.productTourButton, "button button--secondary button--lg")}
+                to="https://www.acryldata.io/tour"
+              >
+                Live Product Tour →
+              </Link>
+              <ScrollingCustomers />
             </div>
-            <div className={clsx(styles.col, "col col--5")}>
-               <DemoForm formId="heroForm" />
+            <div className={clsx(styles.col, styles.hideOnMobile, "col col--5")}>
+              <DemoForm formId="heroForm" />
             </div>
           </div>
         </div>
       </div>
+
+      {isModalOpen && (
+        <DemoFormModal formId="heroFormMobile" handleCloseModal={handleCloseModal} />
+      )}
     </header>
   );
 };

diff --git a/docs-website/src/pages/cloud/Hero/styles.module.scss b/docs-website/src/pages/cloud/Hero/styles.module.scss
@@ -64,11 +64,15 @@
   .productTourButton {
     text-align: center!important;
   }
+
+  .hideOnMobile{
+    display: none;
+  }
 }
 
 @media screen  and (min-width: 1000px){
   .productTourButton {
     padding-left: 0!important;
     margin-left: 0!important;
   }
-}
+}