feat(iceberg): Upgrade Iceberg ingestion source to pyiceberg 0.4.0

metadata-ingestion/docs/sources/iceberg/iceberg_recipe.yml

@@ -2,14 +2,17 @@ source:
   type: "iceberg"
   config:
     env: PROD
-    adls:
-      # Will be translated to https://{account_name}.dfs.core.windows.net
-      account_name: my_adls_account
-      # Can use sas_token or account_key
-      sas_token: "${SAS_TOKEN}"
-      # account_key: "${ACCOUNT_KEY}"
-      container_name: warehouse
-      base_path: iceberg
+    catalog:
+      name: my_iceberg_catalog
+      type: rest
+      # Catalog configuration follows pyiceberg's documentation (https://py.iceberg.apache.org/configuration)
+      config:
+        uri: http://localhost:8181
+        s3.access-key-id: admin
+        s3.secret-access-key: password
+        s3.region: us-east-1
+        warehouse: s3a://warehouse/wh/
+        s3.endpoint: http://localhost:9000
     platform_instance: my_iceberg_catalog
     table_pattern:
       allow:

metadata-ingestion/setup.py

@@ -215,8 +215,8 @@ def get_long_description():
 
 iceberg_common = {
     # Iceberg Python SDK
-    "acryl-iceberg-legacy==0.0.4",
-    "azure-identity==1.10.0",
+    "pyiceberg==0.4.0",
+    "pyarrow>=9.0.0, <13.0.0",
 }
 
 s3_base = {
@@ -462,7 +462,7 @@ def get_long_description():
             "druid",
             "elasticsearch",
             "feast" if sys.version_info >= (3, 8) else None,
-            "iceberg",
+            "iceberg" if sys.version_info >= (3, 8) else None,
             "json-schema",
             "ldap",
             "looker",
@@ -517,7 +517,7 @@ def get_long_description():
             "druid",
             "hana",
             "hive",
-            "iceberg",
+            "iceberg" if sys.version_info >= (3, 8) else None,
             "kafka-connect",
             "ldap",
             "mongodb",
@@ -527,6 +527,7 @@ def get_long_description():
             "redash",
             # "vertica",
         ]
+        if plugin
         for dependency in plugins[plugin]
     ),
 }