feat(iceberg): Upgrade Iceberg ingestion source to pyiceberg 0.4.0 (#…

…8357) Co-authored-by: cccs-Dustin <[email protected]> Co-authored-by: Fokko Driesprong <[email protected]> Co-authored-by: Andrew Sikowitz <[email protected]>
datahub-project · Aug 31, 2023 · 6fe60a2 · 6fe60a2
1 parent a4e7268
commit 6fe60a2
Show file tree

Hide file tree

Showing 45 changed files with 1,653 additions and 2,654 deletions.
diff --git a/metadata-ingestion/docs/sources/iceberg/iceberg_recipe.yml b/metadata-ingestion/docs/sources/iceberg/iceberg_recipe.yml
@@ -2,14 +2,17 @@ source:
   type: "iceberg"
   config:
     env: PROD
-    adls:
-      # Will be translated to https://{account_name}.dfs.core.windows.net
-      account_name: my_adls_account
-      # Can use sas_token or account_key
-      sas_token: "${SAS_TOKEN}"
-      # account_key: "${ACCOUNT_KEY}"
-      container_name: warehouse
-      base_path: iceberg
+    catalog:
+      name: my_iceberg_catalog
+      type: rest
+      # Catalog configuration follows pyiceberg's documentation (https://py.iceberg.apache.org/configuration)
+      config:
+        uri: http://localhost:8181
+        s3.access-key-id: admin
+        s3.secret-access-key: password
+        s3.region: us-east-1
+        warehouse: s3a://warehouse/wh/
+        s3.endpoint: http://localhost:9000
     platform_instance: my_iceberg_catalog
     table_pattern:
       allow:

diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py
@@ -229,8 +229,8 @@ def get_long_description():
 
 iceberg_common = {
     # Iceberg Python SDK
-    "acryl-iceberg-legacy==0.0.4",
-    "azure-identity==1.10.0",
+    "pyiceberg",
+    "pyarrow>=9.0.0, <13.0.0",
 }
 
 s3_base = {
@@ -477,7 +477,7 @@ def get_long_description():
             "druid",
             "elasticsearch",
             "feast" if sys.version_info >= (3, 8) else None,
-            "iceberg",
+            "iceberg" if sys.version_info >= (3, 8) else None,
             "json-schema",
             "ldap",
             "looker",
@@ -530,7 +530,7 @@ def get_long_description():
             "druid",
             "hana",
             "hive",
-            "iceberg",
+            "iceberg" if sys.version_info >= (3, 8) else None,
             "kafka-connect",
             "ldap",
             "mongodb",
@@ -540,6 +540,7 @@ def get_long_description():
             "redash",
             "vertica",
         ]
+        if plugin
         for dependency in plugins[plugin]
     ),
 }

diff --git a/metadata-ingestion/src/datahub/ingestion/source/azure/__init__.py b/metadata-ingestion/src/datahub/ingestion/source/azure/__init__.py
diff --git a/metadata-ingestion/src/datahub/ingestion/source/azure/azure_common.py b/metadata-ingestion/src/datahub/ingestion/source/azure/azure_common.py