diff --git a/docs/iceberg-rest-service.md b/docs/iceberg-rest-service.md index a5760118cc1..72421f0e779 100644 --- a/docs/iceberg-rest-service.md +++ b/docs/iceberg-rest-service.md @@ -14,7 +14,6 @@ The Apache Gravitino Iceberg REST Server follows the [Apache Iceberg REST API sp - Supports the Apache Iceberg REST API defined in Iceberg 1.5, and supports all namespace and table interfaces. The following interfaces are not implemented yet: - token - - view - multi table transaction - pagination - Works as a catalog proxy, supporting `Hive` and `JDBC` as catalog backend. @@ -215,6 +214,15 @@ You must download the corresponding JDBC driver to the `iceberg-rest-server/libs If you want to use a custom Iceberg Catalog as `catalog-backend`, you can add a corresponding jar file to the classpath and load a custom Iceberg Catalog implementation by specifying the `catalog-backend-impl` property. +#### View support + +You could access the view interface if using JDBC backend and enable `jdbc.schema-version` property. + +| Configuration item | Description | Default value | Required | Since Version | +|-------------------------------------------------|--------------------------------------------------------------------------------------------|---------------|----------|---------------| +| `gravitino.iceberg-rest.jdbc.schema-version` | The schema version of JDBC catalog backend, setting to `V1` if supporting view operations. | (none) | NO | 0.7.0 | + + #### Multi catalog support The Gravitino Iceberg REST server supports multiple catalogs and offers a configuration-based catalog management system. diff --git a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/ops/IcebergCatalogWrapper.java b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/ops/IcebergCatalogWrapper.java index 0c7c2914b76..6ff4bf2ce03 100644 --- a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/ops/IcebergCatalogWrapper.java +++ b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/ops/IcebergCatalogWrapper.java @@ -43,9 +43,11 @@ import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.SupportsNamespaces; import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.catalog.ViewCatalog; import org.apache.iceberg.rest.CatalogHandlers; import org.apache.iceberg.rest.requests.CreateNamespaceRequest; import org.apache.iceberg.rest.requests.CreateTableRequest; +import org.apache.iceberg.rest.requests.CreateViewRequest; import org.apache.iceberg.rest.requests.RegisterTableRequest; import org.apache.iceberg.rest.requests.RenameTableRequest; import org.apache.iceberg.rest.requests.UpdateNamespacePropertiesRequest; @@ -55,6 +57,7 @@ import org.apache.iceberg.rest.responses.ListNamespacesResponse; import org.apache.iceberg.rest.responses.ListTablesResponse; import org.apache.iceberg.rest.responses.LoadTableResponse; +import org.apache.iceberg.rest.responses.LoadViewResponse; import org.apache.iceberg.rest.responses.UpdateNamespacePropertiesResponse; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -116,6 +119,13 @@ private void validateNamespace(Optional namespace) { } } + private ViewCatalog getViewCatalog() { + if (!(catalog instanceof ViewCatalog)) { + throw new UnsupportedOperationException(catalog.name() + " is not support view"); + } + return (ViewCatalog) catalog; + } + public CreateNamespaceResponse createNamespace(CreateNamespaceRequest request) { validateNamespace(Optional.of(request.namespace())); return CatalogHandlers.createNamespace(asNamespaceCatalog, request); @@ -203,6 +213,37 @@ public LoadTableResponse updateTable(IcebergTableChange icebergTableChange) { return loadTable(icebergTableChange.getTableIdentifier()); } + public LoadViewResponse createView(Namespace namespace, CreateViewRequest request) { + request.validate(); + return CatalogHandlers.createView(getViewCatalog(), namespace, request); + } + + public LoadViewResponse updateView(TableIdentifier viewIdentifier, UpdateTableRequest request) { + request.validate(); + return CatalogHandlers.updateView(getViewCatalog(), viewIdentifier, request); + } + + public LoadViewResponse loadView(TableIdentifier viewIdentifier) { + return CatalogHandlers.loadView(getViewCatalog(), viewIdentifier); + } + + public void dropView(TableIdentifier viewIdentifier) { + CatalogHandlers.dropView(getViewCatalog(), viewIdentifier); + } + + public void renameView(RenameTableRequest request) { + request.validate(); + CatalogHandlers.renameView(getViewCatalog(), request); + } + + public boolean existView(TableIdentifier viewIdentifier) { + return getViewCatalog().viewExists(viewIdentifier); + } + + public ListTablesResponse listView(Namespace namespace) { + return CatalogHandlers.listViews(getViewCatalog(), namespace); + } + @Override public void close() throws Exception { if (catalog instanceof AutoCloseable) { diff --git a/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/IcebergExceptionMapper.java b/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/IcebergExceptionMapper.java index 95c7bf91ab9..f880f7f7a9f 100644 --- a/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/IcebergExceptionMapper.java +++ b/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/IcebergExceptionMapper.java @@ -32,6 +32,7 @@ import org.apache.iceberg.exceptions.NoSuchIcebergTableException; import org.apache.iceberg.exceptions.NoSuchNamespaceException; import org.apache.iceberg.exceptions.NoSuchTableException; +import org.apache.iceberg.exceptions.NoSuchViewException; import org.apache.iceberg.exceptions.NotAuthorizedException; import org.apache.iceberg.exceptions.ServiceUnavailableException; import org.apache.iceberg.exceptions.UnprocessableEntityException; @@ -57,6 +58,7 @@ public class IcebergExceptionMapper implements ExceptionMapper { .put(NoSuchTableException.class, 404) .put(NoSuchIcebergTableException.class, 404) .put(UnsupportedOperationException.class, 406) + .put(NoSuchViewException.class, 404) .put(AlreadyExistsException.class, 409) .put(CommitFailedException.class, 409) .put(UnprocessableEntityException.class, 422) diff --git a/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/rest/IcebergViewOperations.java b/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/rest/IcebergViewOperations.java new file mode 100644 index 00000000000..3e46257e22b --- /dev/null +++ b/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/rest/IcebergViewOperations.java @@ -0,0 +1,149 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.iceberg.service.rest; + +import com.codahale.metrics.annotation.ResponseMetered; +import com.codahale.metrics.annotation.Timed; +import javax.inject.Inject; +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.Consumes; +import javax.ws.rs.DELETE; +import javax.ws.rs.GET; +import javax.ws.rs.HEAD; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import org.apache.gravitino.iceberg.service.IcebergCatalogWrapperManager; +import org.apache.gravitino.iceberg.service.IcebergRestUtils; +import org.apache.gravitino.metrics.MetricNames; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.rest.RESTUtil; +import org.apache.iceberg.rest.requests.CreateViewRequest; +import org.apache.iceberg.rest.requests.UpdateTableRequest; +import org.apache.iceberg.rest.responses.ListTablesResponse; +import org.apache.iceberg.rest.responses.LoadViewResponse; + +@Path("/v1/{prefix:([^/]*/)?}namespaces/{namespace}/views") +@Consumes(MediaType.APPLICATION_JSON) +@Produces(MediaType.APPLICATION_JSON) +public class IcebergViewOperations { + + private IcebergCatalogWrapperManager icebergCatalogWrapperManager; + + @SuppressWarnings("UnusedVariable") + @Context + private HttpServletRequest httpRequest; + + @Inject + public IcebergViewOperations(IcebergCatalogWrapperManager icebergCatalogWrapperManager) { + this.icebergCatalogWrapperManager = icebergCatalogWrapperManager; + } + + @GET + @Produces(MediaType.APPLICATION_JSON) + @Timed(name = "list-view." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) + @ResponseMetered(name = "list-view", absolute = true) + public Response listView( + @PathParam("prefix") String prefix, @PathParam("namespace") String namespace) { + ListTablesResponse response = + icebergCatalogWrapperManager.getOps(prefix).listView(RESTUtil.decodeNamespace(namespace)); + return IcebergRestUtils.ok(response); + } + + @POST + @Produces(MediaType.APPLICATION_JSON) + @Timed(name = "create-view." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) + @ResponseMetered(name = "create-view", absolute = true) + public Response createView( + @PathParam("prefix") String prefix, + @PathParam("namespace") String namespace, + CreateViewRequest request) { + LoadViewResponse response = + icebergCatalogWrapperManager + .getOps(prefix) + .createView(RESTUtil.decodeNamespace(namespace), request); + return IcebergRestUtils.ok(response); + } + + @GET + @Path("{view}") + @Produces(MediaType.APPLICATION_JSON) + @Timed(name = "load-view." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) + @ResponseMetered(name = "load-view", absolute = true) + public Response loadView( + @PathParam("prefix") String prefix, + @PathParam("namespace") String namespace, + @PathParam("view") String view) { + TableIdentifier viewIdentifier = TableIdentifier.of(RESTUtil.decodeNamespace(namespace), view); + LoadViewResponse response = + icebergCatalogWrapperManager.getOps(prefix).loadView(viewIdentifier); + return IcebergRestUtils.ok(response); + } + + @POST + @Path("{view}") + @Produces(MediaType.APPLICATION_JSON) + @Timed(name = "replace-view." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) + @ResponseMetered(name = "replace-view", absolute = true) + public Response replaceView( + @PathParam("prefix") String prefix, + @PathParam("namespace") String namespace, + @PathParam("view") String view, + UpdateTableRequest request) { + TableIdentifier viewIdentifier = TableIdentifier.of(RESTUtil.decodeNamespace(namespace), view); + LoadViewResponse response = + icebergCatalogWrapperManager.getOps(prefix).updateView(viewIdentifier, request); + return IcebergRestUtils.ok(response); + } + + @DELETE + @Path("{view}") + @Produces(MediaType.APPLICATION_JSON) + @Timed(name = "drop-view." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) + @ResponseMetered(name = "drop-view", absolute = true) + public Response dropView( + @PathParam("prefix") String prefix, + @PathParam("namespace") String namespace, + @PathParam("view") String view) { + TableIdentifier viewIdentifier = TableIdentifier.of(RESTUtil.decodeNamespace(namespace), view); + icebergCatalogWrapperManager.getOps(prefix).dropView(viewIdentifier); + return IcebergRestUtils.noContent(); + } + + @HEAD + @Path("{view}") + @Produces(MediaType.APPLICATION_JSON) + @Timed(name = "view-exists." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) + @ResponseMetered(name = "view-exits", absolute = true) + public Response viewExists( + @PathParam("prefix") String prefix, + @PathParam("namespace") String namespace, + @PathParam("view") String view) { + TableIdentifier tableIdentifier = TableIdentifier.of(RESTUtil.decodeNamespace(namespace), view); + if (icebergCatalogWrapperManager.getOps(prefix).existView(tableIdentifier)) { + return IcebergRestUtils.noContent(); + } else { + return IcebergRestUtils.notExists(); + } + } +} diff --git a/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/rest/IcebergViewRenameOperations.java b/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/rest/IcebergViewRenameOperations.java new file mode 100644 index 00000000000..128689d33be --- /dev/null +++ b/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/rest/IcebergViewRenameOperations.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.iceberg.service.rest; + +import com.codahale.metrics.annotation.ResponseMetered; +import com.codahale.metrics.annotation.Timed; +import javax.inject.Inject; +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.Consumes; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import org.apache.gravitino.iceberg.service.IcebergCatalogWrapperManager; +import org.apache.gravitino.iceberg.service.IcebergRestUtils; +import org.apache.gravitino.metrics.MetricNames; +import org.apache.iceberg.rest.requests.RenameTableRequest; + +@Path("/v1/{prefix:([^/]*/)?}views/rename") +@Consumes(MediaType.APPLICATION_JSON) +@Produces(MediaType.APPLICATION_JSON) +public class IcebergViewRenameOperations { + + @SuppressWarnings("UnusedVariable") + @Context + private HttpServletRequest httpRequest; + + private IcebergCatalogWrapperManager icebergCatalogWrapperManager; + + @Inject + public IcebergViewRenameOperations(IcebergCatalogWrapperManager icebergCatalogWrapperManager) { + this.icebergCatalogWrapperManager = icebergCatalogWrapperManager; + } + + @POST + @Produces(MediaType.APPLICATION_JSON) + @Timed(name = "rename-view." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) + @ResponseMetered(name = "rename-view", absolute = true) + public Response renameView(@PathParam("prefix") String prefix, RenameTableRequest request) { + icebergCatalogWrapperManager.getOps(prefix).renameView(request); + return IcebergRestUtils.noContent(); + } +} diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTJdbcCatalogIT.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTJdbcCatalogIT.java index 1dc758a15c3..d53f8022091 100644 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTJdbcCatalogIT.java +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTJdbcCatalogIT.java @@ -68,6 +68,8 @@ public Map getCatalogConfig() { configMap.put( IcebergConfig.ICEBERG_CONFIG_PREFIX + IcebergConfig.JDBC_INIT_TABLES.getKey(), "true"); + configMap.put(IcebergConfig.ICEBERG_CONFIG_PREFIX + "jdbc.schema-version", "V1"); + configMap.put( IcebergConfig.ICEBERG_CONFIG_PREFIX + IcebergConfig.CATALOG_WAREHOUSE.getKey(), GravitinoITUtils.genRandomName( diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTServiceBaseIT.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTServiceBaseIT.java index e562e2783e4..0ba781cabd8 100644 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTServiceBaseIT.java +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTServiceBaseIT.java @@ -76,6 +76,10 @@ boolean catalogTypeNotMemory() { return !catalogType.equals(IcebergCatalogBackend.MEMORY); } + boolean isSupportsViewCatalog() { + return !catalogType.equals(IcebergCatalogBackend.HIVE); + } + abstract void initEnv(); abstract Map getCatalogConfig(); @@ -175,6 +179,10 @@ protected Map getTableInfo(String tableName) { return convertToStringMap(sql("desc table extended " + tableName)); } + protected Map getViewInfo(String viewName) { + return convertToStringMap(sql("desc extended " + viewName)); + } + protected List getTableColumns(String tableName) { List objects = sql("desc table extended " + tableName); List columns = new ArrayList<>(); diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTServiceIT.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTServiceIT.java index eb196b3a444..9b4900f4d75 100644 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTServiceIT.java +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTServiceIT.java @@ -30,6 +30,7 @@ import org.apache.spark.sql.catalyst.analysis.NamespaceAlreadyExistsException; import org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException; import org.apache.spark.sql.catalyst.analysis.NoSuchTableException; +import org.apache.spark.sql.catalyst.analysis.NoSuchViewException; import org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; @@ -557,4 +558,142 @@ void testRegisterTable() { result = convertToStringMap(sql("SELECT * FROM iceberg_rest_table_test.register_foo2")); Assertions.assertEquals(ImmutableMap.of("1", "a", "2", "b"), result); } + + @Test + @EnabledIf("isSupportsViewCatalog") + void testCreateViewAndDisplayView() { + String originTableName = "iceberg_rest_table_test.create_table_for_view_1"; + String viewName = "iceberg_rest_table_test.test_create_view"; + + sql( + String.format( + "CREATE TABLE %s ( id bigint, data string, ts timestamp) USING iceberg", + originTableName)); + sql(String.format("CREATE VIEW %s AS SELECT * FROM %s", viewName, originTableName)); + + Map viewInfo = getViewInfo(viewName); + Map m = + ImmutableMap.of( + "id", "bigint", + "data", "string", + "ts", "timestamp"); + + checkMapContains(m, viewInfo); + } + + @Test + @EnabledIf("isSupportsViewCatalog") + void testViewProperties() { + String originTableName = "iceberg_rest_table_test.create_table_for_view_2"; + String viewName = "iceberg_rest_table_test.test_create_view_with_properties"; + sql( + String.format( + "CREATE TABLE %s ( id bigint, data string, ts timestamp) USING iceberg", + originTableName)); + + // test create view with properties + sql( + String.format( + "CREATE VIEW %s TBLPROPERTIES ('key1' = 'val1') AS SELECT * FROM %s", + viewName, originTableName)); + + Map viewInfo = getViewInfo(viewName); + Assertions.assertTrue(viewInfo.getOrDefault("View Properties", "").contains("'key1' = 'val1'")); + Assertions.assertFalse( + viewInfo.getOrDefault("View Properties", "").contains("'key2' = 'val2'")); + + // test set properties + sql( + String.format( + "ALTER VIEW %s SET TBLPROPERTIES ('key1' = 'val1', 'key2' = 'val2')", viewName)); + + viewInfo = getViewInfo(viewName); + Assertions.assertTrue(viewInfo.getOrDefault("View Properties", "").contains("'key1' = 'val1'")); + Assertions.assertTrue(viewInfo.getOrDefault("View Properties", "").contains("'key2' = 'val2'")); + + // test unset properties + sql(String.format("ALTER VIEW %s UNSET TBLPROPERTIES ('key1', 'key2')", viewName)); + + viewInfo = getViewInfo(viewName); + Assertions.assertFalse( + viewInfo.getOrDefault("View Properties", "").contains("'key1' = 'val1'")); + Assertions.assertFalse( + viewInfo.getOrDefault("View Properties", "").contains("'key2' = 'val2'")); + } + + @Test + @EnabledIf("isSupportsViewCatalog") + void testDropView() { + String originTableName = "iceberg_rest_table_test.create_table_for_view_3"; + String viewName = "iceberg_rest_table_test.test_drop_view"; + + sql( + String.format( + "CREATE TABLE %s ( id bigint, data string, ts timestamp) USING iceberg", + originTableName)); + sql(String.format("CREATE VIEW %s AS SELECT * FROM %s", viewName, originTableName)); + sql(String.format("DROP VIEW %s", viewName)); + + Assertions.assertThrowsExactly(AnalysisException.class, () -> getViewInfo(viewName)); + Assertions.assertThrowsExactly( + NoSuchViewException.class, () -> sql(String.format("DROP VIEW %s", viewName))); + } + + @Test + @EnabledIf("isSupportsViewCatalog") + void testReplaceView() { + String originTableName = "iceberg_rest_table_test.create_table_for_view_4"; + String viewName = "iceberg_rest_table_test.test_replace_view"; + + sql( + String.format( + "CREATE TABLE %s (id bigint, data string, ts timestamp) USING iceberg", + originTableName)); + sql(String.format("CREATE VIEW %s AS SELECT * FROM %s", viewName, originTableName)); + sql( + String.format( + "CREATE OR REPLACE VIEW %s (updated_id COMMENT 'updated ID') TBLPROPERTIES ('key1' = 'new_val1') AS SELECT id FROM %s", + viewName, originTableName)); + + Map viewInfo = getViewInfo(viewName); + Assertions.assertTrue( + viewInfo.getOrDefault("View Properties", "").contains("'key1' = 'new_val1'")); + Assertions.assertTrue(viewInfo.containsKey("updated_id")); + } + + @Test + @EnabledIf("isSupportsViewCatalog") + void testShowAvailableViews() { + String originTableName = "iceberg_rest_table_test.create_table_for_view_5"; + String viewName1 = "iceberg_rest_table_test.show_available_views_1"; + String viewName2 = "iceberg_rest_table_test.show_available_views_2"; + + sql( + String.format( + "CREATE TABLE %s (id bigint, data string, ts timestamp) USING iceberg", + originTableName)); + sql(String.format("CREATE VIEW %s AS SELECT * FROM %s", viewName1, originTableName)); + sql(String.format("CREATE VIEW %s AS SELECT * FROM %s", viewName2, originTableName)); + + List views = sql("SHOW VIEWS IN iceberg_rest_table_test"); + Assertions.assertEquals(2, views.size()); + } + + @Test + @EnabledIf("isSupportsViewCatalog") + void testShowCreateStatementView() { + String originTableName = "iceberg_rest_table_test.create_table_for_view_6"; + String viewName = "iceberg_rest_table_test.show_create_statement_view"; + + sql( + String.format( + "CREATE TABLE %s (id bigint, data string, ts timestamp) USING iceberg", + originTableName)); + sql(String.format("CREATE VIEW %s AS SELECT * FROM %s", viewName, originTableName)); + + List result = sql(String.format("SHOW CREATE TABLE %s", viewName)); + Assertions.assertEquals(1, result.size()); + Assertions.assertTrue( + Arrays.stream(result.get(0)).findFirst().orElse("").toString().contains(viewName)); + } } diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/IcebergRestTestUtil.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/IcebergRestTestUtil.java index 8bccdab7c56..4fc645132e1 100644 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/IcebergRestTestUtil.java +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/IcebergRestTestUtil.java @@ -44,7 +44,11 @@ public class IcebergRestTestUtil { public static final String UPDATE_NAMESPACE_POSTFIX = "properties"; public static final String TEST_NAMESPACE_NAME = "gravitino-test"; public static final String TABLE_PATH = NAMESPACE_PATH + "/" + TEST_NAMESPACE_NAME + "/tables"; + + public static final String VIEW_PATH = NAMESPACE_PATH + "/" + TEST_NAMESPACE_NAME + "/views"; public static final String RENAME_TABLE_PATH = V_1 + "/tables/rename"; + + public static final String RENAME_VIEW_PATH = V_1 + "/views/rename"; public static final String REPORT_METRICS_POSTFIX = "metrics"; public static final boolean DEBUG_SERVER_LOG_ENABLED = true; diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/IcebergTestBase.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/IcebergTestBase.java index 7d1d80b54e3..03d9a49eb28 100644 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/IcebergTestBase.java +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/IcebergTestBase.java @@ -45,16 +45,30 @@ public Invocation.Builder getRenameTableClientBuilder() { return getIcebergClientBuilder(IcebergRestTestUtil.RENAME_TABLE_PATH, Optional.empty()); } + public Invocation.Builder getRenameViewClientBuilder() { + return getIcebergClientBuilder(IcebergRestTestUtil.RENAME_VIEW_PATH, Optional.empty()); + } + public Invocation.Builder getTableClientBuilder() { return getTableClientBuilder(Optional.empty()); } + public Invocation.Builder getViewClientBuilder() { + return getViewClientBuilder(Optional.empty()); + } + public Invocation.Builder getTableClientBuilder(Optional name) { String path = Joiner.on("/").skipNulls().join(IcebergRestTestUtil.TABLE_PATH, name.orElseGet(() -> null)); return getIcebergClientBuilder(path, Optional.empty()); } + public Invocation.Builder getViewClientBuilder(Optional name) { + String path = + Joiner.on("/").skipNulls().join(IcebergRestTestUtil.VIEW_PATH, name.orElseGet(() -> null)); + return getIcebergClientBuilder(path, Optional.empty()); + } + public Invocation.Builder getReportMetricsClientBuilder(String name) { String path = Joiner.on("/") diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/TestIcebergViewOperations.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/TestIcebergViewOperations.java new file mode 100644 index 00000000000..9ec2dc66f46 --- /dev/null +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/TestIcebergViewOperations.java @@ -0,0 +1,311 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.iceberg.service.rest; + +import com.google.common.collect.ImmutableSet; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import javax.ws.rs.client.Entity; +import javax.ws.rs.core.Application; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import org.apache.iceberg.Schema; +import org.apache.iceberg.UpdateRequirements; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.rest.requests.CreateViewRequest; +import org.apache.iceberg.rest.requests.ImmutableCreateViewRequest; +import org.apache.iceberg.rest.requests.RenameTableRequest; +import org.apache.iceberg.rest.requests.UpdateTableRequest; +import org.apache.iceberg.rest.responses.ListTablesResponse; +import org.apache.iceberg.rest.responses.LoadViewResponse; +import org.apache.iceberg.types.Types; +import org.apache.iceberg.view.ImmutableSQLViewRepresentation; +import org.apache.iceberg.view.ImmutableViewVersion; +import org.apache.iceberg.view.ViewMetadata; +import org.glassfish.jersey.server.ResourceConfig; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +public class TestIcebergViewOperations extends TestIcebergNamespaceOperations { + private static final Schema viewSchema = + new Schema(Types.NestedField.of(1, false, "foo_string", Types.StringType.get())); + + private static final Schema newViewSchema = + new Schema(Types.NestedField.of(2, false, "foo_string1", Types.StringType.get())); + + private static final String VIEW_QUERY = "select 1"; + + @Override + protected Application configure() { + ResourceConfig resourceConfig = + IcebergRestTestUtil.getIcebergResourceConfig(IcebergViewOperations.class); + // create namespace before each view test + resourceConfig.register(IcebergNamespaceOperations.class); + resourceConfig.register(IcebergViewRenameOperations.class); + + return resourceConfig; + } + + @ParameterizedTest + @ValueSource(strings = {"", IcebergRestTestUtil.PREFIX}) + void testListViews(String prefix) { + setUrlPathWithPrefix(prefix); + verifyListViewFail(404); + + verifyCreateNamespaceSucc(IcebergRestTestUtil.TEST_NAMESPACE_NAME); + verifyCreateViewSucc("list_foo1"); + verifyCreateViewSucc("list_foo2"); + verifyLisViewSucc(ImmutableSet.of("list_foo1", "list_foo2")); + } + + @Test + void testCreateView() { + verifyCreateViewFail("create_foo1", 404); + + verifyCreateNamespaceSucc(IcebergRestTestUtil.TEST_NAMESPACE_NAME); + + verifyCreateViewSucc("create_foo1"); + + verifyCreateViewFail("create_foo1", 409); + verifyCreateViewFail("", 400); + } + + @Test + void testLoadView() { + verifyLoadViewFail("load_foo1", 404); + + verifyCreateNamespaceSucc(IcebergRestTestUtil.TEST_NAMESPACE_NAME); + verifyCreateViewSucc("load_foo1"); + verifyLoadViewSucc("load_foo1"); + + verifyLoadViewFail("load_foo2", 404); + } + + @Test + void testReplaceView() { + verifyCreateNamespaceSucc(IcebergRestTestUtil.TEST_NAMESPACE_NAME); + verifyCreateViewSucc("replace_foo1"); + ViewMetadata metadata = getViewMeta("replace_foo1"); + verifyReplaceSucc("replace_foo1", metadata); + + verifyDropViewSucc("replace_foo1"); + verifyUpdateViewFail("replace_foo1", 404, metadata); + + verifyDropNamespaceSucc(IcebergRestTestUtil.TEST_NAMESPACE_NAME); + verifyUpdateViewFail("replace_foo1", 404, metadata); + } + + @Test + void testDropView() { + verifyDropViewFail("drop_foo1", 404); + verifyCreateNamespaceSucc(IcebergRestTestUtil.TEST_NAMESPACE_NAME); + verifyDropViewFail("drop_foo1", 404); + + verifyCreateViewSucc("drop_foo1"); + verifyDropViewSucc("drop_foo1"); + verifyLoadViewFail("drop_foo1", 404); + } + + @Test + void testViewExits() { + verifyViewExistsStatusCode("exists_foo2", 404); + verifyCreateNamespaceSucc(IcebergRestTestUtil.TEST_NAMESPACE_NAME); + verifyViewExistsStatusCode("exists_foo2", 404); + + verifyCreateViewSucc("exists_foo1"); + verifyViewExistsStatusCode("exists_foo1", 204); + verifyLoadViewSucc("exists_foo1"); + } + + @ParameterizedTest + @ValueSource(strings = {"", IcebergRestTestUtil.PREFIX}) + void testRenameTable(String prefix) { + setUrlPathWithPrefix(prefix); + // namespace not exits + verifyRenameViewFail("rename_foo1", "rename_foo3", 404); + + verifyCreateNamespaceSucc(IcebergRestTestUtil.TEST_NAMESPACE_NAME); + verifyCreateViewSucc("rename_foo1"); + // rename + verifyRenameViewSucc("rename_foo1", "rename_foo2"); + verifyLoadViewFail("rename_foo1", 404); + verifyLoadViewSucc("rename_foo2"); + + // source view not exists + verifyRenameViewFail("rename_foo1", "rename_foo3", 404); + + // dest view exists + verifyCreateViewSucc("rename_foo3"); + verifyRenameViewFail("rename_foo2", "rename_foo3", 409); + } + + private Response doCreateView(String name) { + CreateViewRequest createViewRequest = + ImmutableCreateViewRequest.builder() + .name(name) + .schema(viewSchema) + .viewVersion( + ImmutableViewVersion.builder() + .versionId(1) + .timestampMillis(System.currentTimeMillis()) + .schemaId(1) + .defaultNamespace(Namespace.of(IcebergRestTestUtil.TEST_NAMESPACE_NAME)) + .addRepresentations( + ImmutableSQLViewRepresentation.builder() + .sql(VIEW_QUERY) + .dialect("spark") + .build()) + .build()) + .build(); + return getViewClientBuilder() + .post(Entity.entity(createViewRequest, MediaType.APPLICATION_JSON_TYPE)); + } + + private Response doLoadView(String name) { + return getViewClientBuilder(Optional.of(name)).get(); + } + + private void verifyLoadViewSucc(String name) { + Response response = doLoadView(name); + Assertions.assertEquals(Response.Status.OK.getStatusCode(), response.getStatus()); + + LoadViewResponse loadViewResponse = response.readEntity(LoadViewResponse.class); + Assertions.assertEquals(viewSchema.columns(), loadViewResponse.metadata().schema().columns()); + } + + private void verifyCreateViewFail(String name, int status) { + Response response = doCreateView(name); + Assertions.assertEquals(status, response.getStatus()); + } + + private void verifyCreateViewSucc(String name) { + Response response = doCreateView(name); + Assertions.assertEquals(Response.Status.OK.getStatusCode(), response.getStatus()); + LoadViewResponse loadViewResponse = response.readEntity(LoadViewResponse.class); + Schema schema = loadViewResponse.metadata().schema(); + Assertions.assertEquals(schema.columns(), viewSchema.columns()); + } + + private void verifyLoadViewFail(String name, int status) { + Response response = doLoadView(name); + Assertions.assertEquals(status, response.getStatus()); + } + + private void verifyReplaceSucc(String name, ViewMetadata base) { + Response response = doReplaceView(name, base); + Assertions.assertEquals(Response.Status.OK.getStatusCode(), response.getStatus()); + LoadViewResponse loadViewResponse = response.readEntity(LoadViewResponse.class); + Assertions.assertEquals( + newViewSchema.columns(), loadViewResponse.metadata().schema().columns()); + } + + private Response doReplaceView(String name, ViewMetadata base) { + ViewMetadata.Builder builder = + ViewMetadata.buildFrom(base).setCurrentVersion(base.currentVersion(), newViewSchema); + ViewMetadata replacement = builder.build(); + UpdateTableRequest updateTableRequest = + UpdateTableRequest.create( + null, + UpdateRequirements.forReplaceView(base, replacement.changes()), + replacement.changes()); + return getViewClientBuilder(Optional.of(name)) + .post(Entity.entity(updateTableRequest, MediaType.APPLICATION_JSON_TYPE)); + } + + private ViewMetadata getViewMeta(String viewName) { + Response response = doLoadView(viewName); + LoadViewResponse loadViewResponse = response.readEntity(LoadViewResponse.class); + return loadViewResponse.metadata(); + } + + private void verifyUpdateViewFail(String name, int status, ViewMetadata base) { + Response response = doReplaceView(name, base); + Assertions.assertEquals(status, response.getStatus()); + } + + private void verifyDropViewSucc(String name) { + Response response = doDropView(name); + Assertions.assertEquals(Response.Status.NO_CONTENT.getStatusCode(), response.getStatus()); + } + + private Response doDropView(String name) { + return getViewClientBuilder(Optional.of(name)).delete(); + } + + private void verifyDropViewFail(String name, int status) { + Response response = doDropView(name); + Assertions.assertEquals(status, response.getStatus()); + } + + private void verifyViewExistsStatusCode(String name, int status) { + Response response = doViewExists(name); + Assertions.assertEquals(status, response.getStatus()); + } + + private Response doViewExists(String name) { + return getViewClientBuilder(Optional.of(name)).head(); + } + + private void verifyListViewFail(int status) { + Response response = doListView(); + Assertions.assertEquals(status, response.getStatus()); + } + + private Response doListView() { + return getViewClientBuilder().get(); + } + + private void verifyLisViewSucc(Set expectedTableNames) { + Response response = doListView(); + Assertions.assertEquals(Response.Status.OK.getStatusCode(), response.getStatus()); + ListTablesResponse listTablesResponse = response.readEntity(ListTablesResponse.class); + Set tableNames = + listTablesResponse.identifiers().stream() + .map(identifier -> identifier.name()) + .collect(Collectors.toSet()); + Assertions.assertEquals(expectedTableNames, tableNames); + } + + private void verifyRenameViewFail(String source, String dest, int status) { + Response response = doRenameView(source, dest); + Assertions.assertEquals(status, response.getStatus()); + } + + private Response doRenameView(String source, String dest) { + RenameTableRequest renameTableRequest = + RenameTableRequest.builder() + .withSource( + TableIdentifier.of(Namespace.of(IcebergRestTestUtil.TEST_NAMESPACE_NAME), source)) + .withDestination( + TableIdentifier.of(Namespace.of(IcebergRestTestUtil.TEST_NAMESPACE_NAME), dest)) + .build(); + return getRenameViewClientBuilder() + .post(Entity.entity(renameTableRequest, MediaType.APPLICATION_JSON_TYPE)); + } + + private void verifyRenameViewSucc(String source, String dest) { + Response response = doRenameView(source, dest); + Assertions.assertEquals(Response.Status.NO_CONTENT.getStatusCode(), response.getStatus()); + } +}