Merge branch 'main' into CLI_VIXb

.github/workflows/backend-integration-test.yml

@@ -90,4 +90,4 @@ jobs:
       architecture: ${{ matrix.architecture }}
       java-version: ${{ matrix.java-version }}
       backend: ${{ matrix.backend }}
-      test-mode: ${{ matrix.test-mode }}
+      test-mode: ${{ matrix.test-mode }}

.github/workflows/build.yml

@@ -28,6 +28,7 @@ jobs:
               - authorizations/**
               - bin/**
               - catalogs/**
+              - clients/cli/**
               - clients/client-java/**
               - clients/client-java-runtime/**
               - clients/filesystem-hadoop3/**

README.md

@@ -30,20 +30,20 @@
 
 ## Introduction
 
-Apache Gravitino is a high-performance, geo-distributed, and federated metadata lake. It manages metadata directly in different sources, types, and regions and provides users with unified metadata access for data and AI assets.
+Apache Gravitino is a high-performance, geo-distributed, and federated metadata lake. It manages metadata directly in different sources, types, and regions, providing users with unified metadata access for data and AI assets.
 
 ![Gravitino Architecture](docs/assets/gravitino-architecture.png)
 
 Gravitino aims to provide several key features:
 
-* Single Source of Truth for multi-regional data with geo-distributed architecture support.
-* Unified Data and AI asset management for both users and engines.
+* Single source of truth for multi-regional data with geo-distributed architecture support.
+* Unified data and AI asset management for both users and engines.
 * Security in one place, centralizing the security for different sources.
 * Built-in data management and data access management.
 
 ## Contributing to Apache Gravitino
 
-Gravitino is open source software available under the Apache 2.0 license. For information on how to contribute to Gravitino, please see the [Contribution guidelines](https://gravitino.apache.org/contrib/).
+Gravitino is open source software available under the Apache 2.0 license. For information on contributing to Gravitino, please see the [Contribution guidelines](https://gravitino.apache.org/contrib/).
 
 ## Online documentation
 
@@ -83,14 +83,13 @@ Please see [How to build Gravitino](https://gravitino.apache.org/docs/latest/how
 
 ### Use Gravitino playground
 
-This is the most recommended way: Gravitino provides a docker-compose based playground to quickly experience the whole system together with other components. Clone or download the [Gravitino playground repository](https://github.com/apache/gravitino-playground) and then follow the [README](https://github.com/apache/gravitino-playground/blob/main/README.md), you will have all then.
+This is the recommended approach. Gravitino provides a docker-compose-based playground where you can experience a whole system alongside other components. Clone or download the [Gravitino playground repository](https://github.com/apache/gravitino-playground) and then follow the [README](https://github.com/apache/gravitino-playground/blob/main/README.md), to get everything running.
 
 ### Configure and start Gravitino server in local
 
-If you want to start Gravitino in your machine, download a binary package from the [download page](https://gravitino.apache.org/downloads), and then decompressed the package.
+To start Gravitino on your machine, download a binary package from the [download page](https://gravitino.apache.org/downloads) and decompress the package.
 
-Before starting the Gravitino server, please configure the Gravitino server configuration file. The
-configuration file, `gravitino.conf`, is in the `conf` directory and follows the standard property file format. You can modify the configuration within this file.
+Before starting the Gravitino server, configure its settings by editing the `gravitino.conf` file located in the `conf` directory. This file follows the standard properties file format, allowing you to modify the server configuration as needed.
 
 To start the Gravitino server, please run:
 
@@ -114,11 +113,11 @@ And press `CTRL+C` to stop the Gravitino server.
 
 ### Gravitino Iceberg REST catalog service
 
-Gravitino provides Iceberg REST catalog service to manage Iceberg. For more details, please refer to [Gravitino Iceberg REST catalog service](https://gravitino.apache.org/docs/latest/iceberg-rest-service/).
+Gravitino provides Iceberg REST catalog service to manage Iceberg efficiently. For more details, refer to [Gravitino Iceberg REST catalog service](https://gravitino.apache.org/docs/latest/iceberg-rest-service/).
 
 ### Using Trino with Apache Gravitino
 
-Gravitino provides a Trino connector to access the metadata in Gravitino. To use Trino with Gravitino, please follow the [trino-gravitino-connector doc](https://gravitino.apache.org/docs/latest/trino-connector/index/).
+Gravitino provides a Trino connector for accessing metadata within Gravitino. To use Trino with Gravitino, please follow the [trino-gravitino-connector doc](https://gravitino.apache.org/docs/latest/trino-connector/index/).
 
 ## Development guide
 

authorizations/authorization-ranger/build.gradle.kts

@@ -26,9 +26,10 @@ plugins {
 
 val scalaVersion: String = project.properties["scalaVersion"] as? String ?: extra["defaultScalaVersion"].toString()
 val sparkVersion: String = libs.versions.spark35.get()
-val kyuubiVersion: String = libs.versions.kyuubi4spark35.get()
+val kyuubiVersion: String = libs.versions.kyuubi4paimon.get()
 val sparkMajorVersion: String = sparkVersion.substringBeforeLast(".")
 val icebergVersion: String = libs.versions.iceberg4spark.get()
+val paimonVersion: String = libs.versions.paimon.get()
 
 dependencies {
   implementation(project(":api")) {
@@ -86,7 +87,7 @@ dependencies {
     exclude("io.dropwizard.metrics")
     exclude("org.rocksdb")
   }
-  testImplementation("org.apache.kyuubi:kyuubi-spark-authz_$scalaVersion:$kyuubiVersion") {
+  testImplementation("org.apache.kyuubi:kyuubi-spark-authz-shaded_$scalaVersion:$kyuubiVersion") {
     exclude("com.sun.jersey")
   }
   testImplementation(libs.hadoop3.client)
@@ -100,6 +101,7 @@ dependencies {
     exclude("io.netty")
   }
   testImplementation("org.apache.iceberg:iceberg-spark-runtime-${sparkMajorVersion}_$scalaVersion:$icebergVersion")
+  testImplementation("org.apache.paimon:paimon-spark-$sparkMajorVersion:$paimonVersion")
 }
 
 tasks {

authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorization.java

@@ -34,6 +34,7 @@ protected AuthorizationPlugin newPlugin(String catalogProvider, Map<String, Stri
     switch (catalogProvider) {
       case "hive":
       case "lakehouse-iceberg":
+      case "lakehouse-paimon":
         return RangerAuthorizationHadoopSQLPlugin.getInstance(config);
       default:
         throw new IllegalArgumentException("Unknown catalog provider: " + catalogProvider);

authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHadoopSQLPlugin.java

@@ -35,7 +35,7 @@
 import org.apache.gravitino.authorization.Privilege;
 import org.apache.gravitino.authorization.SecurableObject;
 import org.apache.gravitino.authorization.SecurableObjects;
-import org.apache.gravitino.authorization.ranger.RangerPrivileges.RangerHivePrivilege;
+import org.apache.gravitino.authorization.ranger.RangerPrivileges.RangerHadoopSQLPrivilege;
 import org.apache.gravitino.authorization.ranger.reference.RangerDefines.PolicyResource;
 import org.apache.gravitino.exceptions.AuthorizationPluginException;
 import org.slf4j.Logger;
@@ -96,26 +96,28 @@ public void validateRangerMetadataObject(List<String> names, RangerMetadataObjec
   public Map<Privilege.Name, Set<RangerPrivilege>> privilegesMappingRule() {
     return ImmutableMap.of(
         Privilege.Name.CREATE_CATALOG,
-        ImmutableSet.of(RangerHivePrivilege.CREATE),
+        ImmutableSet.of(RangerHadoopSQLPrivilege.CREATE),
         Privilege.Name.USE_CATALOG,
-        ImmutableSet.of(RangerHivePrivilege.SELECT),
+        ImmutableSet.of(RangerHadoopSQLPrivilege.SELECT),
         Privilege.Name.CREATE_SCHEMA,
-        ImmutableSet.of(RangerHivePrivilege.CREATE),
+        ImmutableSet.of(RangerHadoopSQLPrivilege.CREATE),
         Privilege.Name.USE_SCHEMA,
-        ImmutableSet.of(RangerHivePrivilege.SELECT),
+        ImmutableSet.of(RangerHadoopSQLPrivilege.SELECT),
         Privilege.Name.CREATE_TABLE,
-        ImmutableSet.of(RangerHivePrivilege.CREATE),
+        ImmutableSet.of(RangerHadoopSQLPrivilege.CREATE),
         Privilege.Name.MODIFY_TABLE,
         ImmutableSet.of(
-            RangerHivePrivilege.UPDATE, RangerHivePrivilege.ALTER, RangerHivePrivilege.WRITE),
+            RangerHadoopSQLPrivilege.UPDATE,
+            RangerHadoopSQLPrivilege.ALTER,
+            RangerHadoopSQLPrivilege.WRITE),
         Privilege.Name.SELECT_TABLE,
-        ImmutableSet.of(RangerHivePrivilege.READ, RangerHivePrivilege.SELECT));
+        ImmutableSet.of(RangerHadoopSQLPrivilege.READ, RangerHadoopSQLPrivilege.SELECT));
   }
 
   @Override
   /** Set the default owner rule. */
   public Set<RangerPrivilege> ownerMappingRule() {
-    return ImmutableSet.of(RangerHivePrivilege.ALL);
+    return ImmutableSet.of(RangerHadoopSQLPrivilege.ALL);
   }
 
   @Override

authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerPrivileges.java

@@ -25,7 +25,7 @@
 
 public class RangerPrivileges {
   /** Ranger Hive privileges enumeration. */
-  public enum RangerHivePrivilege implements RangerPrivilege {
+  public enum RangerHadoopSQLPrivilege implements RangerPrivilege {
     ALL("all"),
     SELECT("select"),
     UPDATE("update"),
@@ -41,7 +41,7 @@ public enum RangerHivePrivilege implements RangerPrivilege {
 
     private final String name; // Access a type in the Ranger policy item
 
-    RangerHivePrivilege(String name) {
+    RangerHadoopSQLPrivilege(String name) {
       this.name = name;
     }
 
@@ -117,7 +117,7 @@ public boolean equalsTo(String value) {
 
   static List<Class<? extends Enum<? extends RangerPrivilege>>> allRangerPrivileges =
       Lists.newArrayList(
-          RangerPrivileges.RangerHivePrivilege.class, RangerPrivileges.RangerHdfsPrivilege.class);
+          RangerHadoopSQLPrivilege.class, RangerPrivileges.RangerHdfsPrivilege.class);
 
   public static RangerPrivilege valueOf(String name) {
     Preconditions.checkArgument(name != null, "Privilege name string cannot be null!");

authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerBaseE2EIT.java

@@ -193,7 +193,7 @@ protected static void waitForUpdatingPolicies() throws InterruptedException {
 
   protected abstract void useCatalog() throws InterruptedException;
 
-  protected abstract void checkHaveNoPrivileges();
+  protected abstract void checkWithoutPrivileges();
 
   protected abstract void testAlterTable();
 
@@ -269,7 +269,7 @@ void testCreateTable() throws InterruptedException {
         AccessControlException.class, () -> sparkSession.sql(SQL_SELECT_TABLE).collectAsList());
 
     // Clean up
-    catalog.asTableCatalog().dropTable(NameIdentifier.of(schemaName, tableName));
+    catalog.asTableCatalog().purgeTable(NameIdentifier.of(schemaName, tableName));
     catalog.asSchemas().dropSchema(schemaName, false);
     metalake.deleteRole(createTableRole);
     metalake.deleteRole(createSchemaRole);
@@ -323,10 +323,10 @@ void testReadWriteTableWithMetalakeLevelRole() throws InterruptedException {
     // case 7: If we don't have the role, we can't insert and select from data.
     metalake.deleteRole(readWriteRole);
     waitForUpdatingPolicies();
-    checkHaveNoPrivileges();
+    checkWithoutPrivileges();
 
     // Clean up
-    catalog.asTableCatalog().dropTable(NameIdentifier.of(schemaName, tableName));
+    catalog.asTableCatalog().purgeTable(NameIdentifier.of(schemaName, tableName));
     catalog.asSchemas().dropSchema(schemaName, false);
   }
 
@@ -387,10 +387,10 @@ void testReadWriteTableWithTableLevelRole() throws InterruptedException {
     // case 7: If we don't have the role, we can't insert and select from data.
     metalake.deleteRole(roleName);
     waitForUpdatingPolicies();
-    checkHaveNoPrivileges();
+    checkWithoutPrivileges();
 
     // Clean up
-    catalog.asTableCatalog().dropTable(NameIdentifier.of(schemaName, tableName));
+    catalog.asTableCatalog().purgeTable(NameIdentifier.of(schemaName, tableName));
     catalog.asSchemas().dropSchema(schemaName, false);
   }
 
@@ -441,10 +441,10 @@ void testReadOnlyTable() throws InterruptedException {
     // case 7: If we don't have the role, we can't insert and select from data.
     metalake.deleteRole(readOnlyRole);
     waitForUpdatingPolicies();
-    checkHaveNoPrivileges();
+    checkWithoutPrivileges();
 
     // Clean up
-    catalog.asTableCatalog().dropTable(NameIdentifier.of(schemaName, tableName));
+    catalog.asTableCatalog().purgeTable(NameIdentifier.of(schemaName, tableName));
     catalog.asSchemas().dropSchema(schemaName, false);
   }
 
@@ -496,10 +496,10 @@ void testWriteOnlyTable() throws InterruptedException {
     // case 7: If we don't have the role, we can't insert and select from data.
     metalake.deleteRole(writeOnlyRole);
     waitForUpdatingPolicies();
-    checkHaveNoPrivileges();
+    checkWithoutPrivileges();
 
     // Clean up
-    catalog.asTableCatalog().dropTable(NameIdentifier.of(schemaName, tableName));
+    catalog.asTableCatalog().purgeTable(NameIdentifier.of(schemaName, tableName));
     catalog.asSchemas().dropSchema(schemaName, false);
   }
 
@@ -529,7 +529,8 @@ void testCreateAllPrivilegesRole() throws InterruptedException {
                 Privileges.ModifyTable.allow(),
                 Privileges.ManageUsers.allow(),
                 Privileges.ManageGroups.allow(),
-                Privileges.CreateRole.allow()));
+                Privileges.CreateRole.allow(),
+                Privileges.ManageGrants.allow()));
     metalake.createRole(roleName, Collections.emptyMap(), Lists.newArrayList(securableObject));
 
     // Granted this role to the spark execution user `HADOOP_USER_NAME`
@@ -546,7 +547,7 @@ void testCreateAllPrivilegesRole() throws InterruptedException {
     sparkSession.sql(SQL_CREATE_TABLE);
 
     // Clean up
-    catalog.asTableCatalog().dropTable(NameIdentifier.of(schemaName, tableName));
+    catalog.asTableCatalog().purgeTable(NameIdentifier.of(schemaName, tableName));
     catalog.asSchemas().dropSchema(schemaName, false);
     metalake.deleteRole(roleName);
   }
@@ -689,7 +690,7 @@ void testRenameMetadataObject() throws InterruptedException {
     sparkSession.sql(SQL_RENAME_BACK_TABLE);
 
     // Clean up
-    catalog.asTableCatalog().dropTable(NameIdentifier.of(schemaName, tableName));
+    catalog.asTableCatalog().purgeTable(NameIdentifier.of(schemaName, tableName));
     catalog.asSchemas().dropSchema(schemaName, false);
     metalake.deleteRole(roleName);
   }
@@ -738,7 +739,7 @@ void testRenameMetadataObjectPrivilege() throws InterruptedException {
     sparkSession.sql(SQL_INSERT_TABLE);
 
     // Clean up
-    catalog.asTableCatalog().dropTable(NameIdentifier.of(schemaName, tableName));
+    catalog.asTableCatalog().purgeTable(NameIdentifier.of(schemaName, tableName));
     catalog.asSchemas().dropSchema(schemaName, false);
     metalake.deleteRole(roleName);
   }
@@ -773,7 +774,7 @@ void testChangeOwner() throws InterruptedException {
     metalake.deleteRole(helperRole);
     waitForUpdatingPolicies();
 
-    checkHaveNoPrivileges();
+    checkWithoutPrivileges();
 
     // case 2. user is the  table owner
     MetadataObject tableObject =
@@ -786,7 +787,7 @@ void testChangeOwner() throws InterruptedException {
     checkTableAllPrivilegesExceptForCreating();
 
     // Delete Gravitino's meta data
-    catalog.asTableCatalog().dropTable(NameIdentifier.of(schemaName, tableName));
+    catalog.asTableCatalog().purgeTable(NameIdentifier.of(schemaName, tableName));
     waitForUpdatingPolicies();
 
     // Fail to create the table
@@ -853,7 +854,7 @@ void testChangeOwner() throws InterruptedException {
     sparkSession.sql(SQL_DROP_SCHEMA);
 
     // Clean up
-    catalog.asTableCatalog().dropTable(NameIdentifier.of(schemaName, tableName));
+    catalog.asTableCatalog().purgeTable(NameIdentifier.of(schemaName, tableName));
     catalog.asSchemas().dropSchema(schemaName, false);
   }
 
@@ -914,7 +915,7 @@ void testAllowUseSchemaPrivilege() throws InterruptedException {
         1, rows2.stream().filter(row -> row.getString(0).equals(schemaName)).count());
 
     // Clean up
-    catalog.asTableCatalog().dropTable(NameIdentifier.of(schemaName, tableName));
+    catalog.asTableCatalog().purgeTable(NameIdentifier.of(schemaName, tableName));
     catalog.asSchemas().dropSchema(schemaName, false);
     metalake.revokeRolesFromUser(Lists.newArrayList(roleName), userName1);
     metalake.deleteRole(roleName);

authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveE2EIT.java

@@ -120,7 +120,7 @@ protected void useCatalog() throws InterruptedException {
   }
 
   @Override
-  protected void checkHaveNoPrivileges() {
+  protected void checkWithoutPrivileges() {
     Assertions.assertThrows(AccessControlException.class, () -> sparkSession.sql(SQL_INSERT_TABLE));
     Assertions.assertThrows(
         AccessControlException.class, () -> sparkSession.sql(SQL_SELECT_TABLE).collectAsList());

authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveIT.java

@@ -348,7 +348,7 @@ public void testFindManagedPolicy() {
             RangerMetadataObject.Type.TABLE,
             ImmutableSet.of(
                 new RangerPrivileges.RangerHivePrivilegeImpl(
-                    RangerPrivileges.RangerHivePrivilege.ALL, Privilege.Condition.ALLOW)));
+                    RangerPrivileges.RangerHadoopSQLPrivilege.ALL, Privilege.Condition.ALLOW)));
     Assertions.assertNull(rangerHelper.findManagedPolicy(rangerSecurableObject));
 
     // Add a policy for `db3.tab1`
@@ -398,7 +398,7 @@ static void createHivePolicy(
     policyItem.setAccesses(
         Arrays.asList(
             new RangerPolicy.RangerPolicyItemAccess(
-                RangerPrivileges.RangerHivePrivilege.SELECT.toString())));
+                RangerPrivileges.RangerHadoopSQLPrivilege.SELECT.toString())));
     RangerITEnv.updateOrCreateRangerPolicy(
         RangerDefines.SERVICE_TYPE_HIVE,
         RangerITEnv.RANGER_HIVE_REPO_NAME,

authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerITEnv.java

@@ -212,7 +212,7 @@ static void allowAnyoneAccessInformationSchema() {
     policyItem.setAccesses(
         Arrays.asList(
             new RangerPolicy.RangerPolicyItemAccess(
-                RangerPrivileges.RangerHivePrivilege.SELECT.toString())));
+                RangerPrivileges.RangerHadoopSQLPrivilege.SELECT.toString())));
     updateOrCreateRangerPolicy(
         RANGER_HIVE_TYPE,
         RANGER_HIVE_REPO_NAME,

authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerIcebergE2EIT.java

@@ -90,7 +90,7 @@ public void startIntegrationTest() throws Exception {
     sparkSession =
         SparkSession.builder()
             .master("local[1]")
-            .appName("Ranger Hive E2E integration test")
+            .appName("Ranger Iceberg E2E integration test")
             .config("spark.sql.catalog.iceberg", "org.apache.iceberg.spark.SparkCatalog")
             .config("spark.sql.catalog.iceberg.type", "hive")
             .config("spark.sql.catalog.iceberg.uri", HIVE_METASTORE_URIS)
@@ -147,7 +147,7 @@ protected void checkDeleteSQLWithWritePrivileges() {
   }
 
   @Override
-  protected void checkHaveNoPrivileges() {
+  protected void checkWithoutPrivileges() {
     Assertions.assertThrows(AccessControlException.class, () -> sparkSession.sql(SQL_INSERT_TABLE));
     Assertions.assertThrows(
         AccessControlException.class, () -> sparkSession.sql(SQL_SELECT_TABLE).collectAsList());