From c9fd67918ddaff9dd9db63c175e735325177a4db Mon Sep 17 00:00:00 2001 From: theoryxu Date: Fri, 13 Sep 2024 17:49:45 +0800 Subject: [PATCH 01/13] [#4370]feat(iceberg): support view interface for Iceberg REST server --- .../iceberg/common/ops/IcebergTableOps.java | 41 +++++ .../service/rest/IcebergViewOperations.java | 144 ++++++++++++++++++ 2 files changed, 185 insertions(+) create mode 100644 iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/rest/IcebergViewOperations.java diff --git a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/ops/IcebergTableOps.java b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/ops/IcebergTableOps.java index aaed61bfdc2..84048c13384 100644 --- a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/ops/IcebergTableOps.java +++ b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/ops/IcebergTableOps.java @@ -43,9 +43,11 @@ import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.SupportsNamespaces; import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.catalog.ViewCatalog; import org.apache.iceberg.rest.CatalogHandlers; import org.apache.iceberg.rest.requests.CreateNamespaceRequest; import org.apache.iceberg.rest.requests.CreateTableRequest; +import org.apache.iceberg.rest.requests.CreateViewRequest; import org.apache.iceberg.rest.requests.RegisterTableRequest; import org.apache.iceberg.rest.requests.RenameTableRequest; import org.apache.iceberg.rest.requests.UpdateNamespacePropertiesRequest; @@ -55,6 +57,7 @@ import org.apache.iceberg.rest.responses.ListNamespacesResponse; import org.apache.iceberg.rest.responses.ListTablesResponse; import org.apache.iceberg.rest.responses.LoadTableResponse; +import org.apache.iceberg.rest.responses.LoadViewResponse; import org.apache.iceberg.rest.responses.UpdateNamespacePropertiesResponse; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -116,6 +119,13 @@ private void validateNamespace(Optional namespace) { } } + private ViewCatalog getViewCatalog() { + if (!(catalog instanceof ViewCatalog)) { + throw new UnsupportedOperationException(catalog.name() + " is not support view"); + } + return (ViewCatalog) catalog; + } + public CreateNamespaceResponse createNamespace(CreateNamespaceRequest request) { validateNamespace(Optional.of(request.namespace())); return CatalogHandlers.createNamespace(asNamespaceCatalog, request); @@ -203,6 +213,37 @@ public LoadTableResponse updateTable(IcebergTableChange icebergTableChange) { return loadTable(icebergTableChange.getTableIdentifier()); } + public LoadViewResponse createView(Namespace namespace, CreateViewRequest request) { + request.validate(); + return CatalogHandlers.createView(getViewCatalog(), namespace, request); + } + + public LoadViewResponse updateView(TableIdentifier viewIdentifier, UpdateTableRequest request) { + request.validate(); + return CatalogHandlers.updateView(getViewCatalog(), viewIdentifier, request); + } + + public LoadViewResponse loadView(TableIdentifier viewIdentifier) { + return CatalogHandlers.loadView(getViewCatalog(), viewIdentifier); + } + + public void dropView(TableIdentifier viewIdentifier) { + CatalogHandlers.dropView(getViewCatalog(), viewIdentifier); + } + + public void renameView(RenameTableRequest request) { + request.validate(); + CatalogHandlers.renameView(getViewCatalog(), request); + } + + public boolean existView(TableIdentifier viewIdentifier) { + return getViewCatalog().viewExists(viewIdentifier); + } + + public ListTablesResponse listView(Namespace namespace) { + return CatalogHandlers.listViews(getViewCatalog(), namespace); + } + @Override public void close() throws Exception { if (catalog instanceof AutoCloseable) { diff --git a/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/rest/IcebergViewOperations.java b/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/rest/IcebergViewOperations.java new file mode 100644 index 00000000000..6ae03433abc --- /dev/null +++ b/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/rest/IcebergViewOperations.java @@ -0,0 +1,144 @@ +package org.apache.gravitino.iceberg.service.rest; + +import com.codahale.metrics.annotation.ResponseMetered; +import com.codahale.metrics.annotation.Timed; +import javax.inject.Inject; +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.Consumes; +import javax.ws.rs.DELETE; +import javax.ws.rs.GET; +import javax.ws.rs.HEAD; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import org.apache.gravitino.iceberg.service.IcebergRestUtils; +import org.apache.gravitino.iceberg.service.IcebergTableOpsManager; +import org.apache.gravitino.metrics.MetricNames; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.rest.RESTUtil; +import org.apache.iceberg.rest.requests.CreateViewRequest; +import org.apache.iceberg.rest.requests.RenameTableRequest; +import org.apache.iceberg.rest.requests.UpdateTableRequest; +import org.apache.iceberg.rest.responses.ListTablesResponse; +import org.apache.iceberg.rest.responses.LoadViewResponse; + +@Path("/v1/{prefix:([^/]*/)?}namespaces/{namespace}/views") +@Consumes(MediaType.APPLICATION_JSON) +@Produces(MediaType.APPLICATION_JSON) +public class IcebergViewOperations { + + private IcebergTableOpsManager icebergTableOpsManager; + + @SuppressWarnings("UnusedVariable") + @Context + private HttpServletRequest httpRequest; + + @Inject + public IcebergViewOperations(IcebergTableOpsManager icebergTableOpsManager) { + this.icebergTableOpsManager = icebergTableOpsManager; + } + + @GET + @Produces(MediaType.APPLICATION_JSON) + @Timed(name = "list-view." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) + @ResponseMetered(name = "list-view", absolute = true) + public Response listView( + @PathParam("prefix") String prefix, @PathParam("namespace") String namespace) { + ListTablesResponse response = + icebergTableOpsManager.getOps(prefix).listView(RESTUtil.decodeNamespace(namespace)); + return IcebergRestUtils.ok(response); + } + + @POST + @Produces(MediaType.APPLICATION_JSON) + @Timed(name = "create-view." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) + @ResponseMetered(name = "create-view", absolute = true) + public Response createView( + @PathParam("prefix") String prefix, + @PathParam("namespace") String namespace, + CreateViewRequest request) { + LoadViewResponse response = + icebergTableOpsManager + .getOps(prefix) + .createView(RESTUtil.decodeNamespace(namespace), request); + return IcebergRestUtils.ok(response); + } + + @GET + @Path("{view}") + @Produces(MediaType.APPLICATION_JSON) + @Timed(name = "load-view." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) + @ResponseMetered(name = "load-view", absolute = true) + public Response loadView( + @PathParam("prefix") String prefix, + @PathParam("namespace") String namespace, + @PathParam("view") String view) { + TableIdentifier viewIdentifier = TableIdentifier.of(RESTUtil.decodeNamespace(namespace), view); + LoadViewResponse response = icebergTableOpsManager.getOps(prefix).loadView(viewIdentifier); + return IcebergRestUtils.ok(response); + } + + @POST + @Path("{view}") + @Produces(MediaType.APPLICATION_JSON) + @Timed(name = "replace-view." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) + @ResponseMetered(name = "replace-view", absolute = true) + public Response replaceView( + @PathParam("prefix") String prefix, + @PathParam("namespace") String namespace, + @PathParam("view") String view, + UpdateTableRequest request) { + TableIdentifier viewIdentifier = TableIdentifier.of(RESTUtil.decodeNamespace(namespace), view); + LoadViewResponse response = + icebergTableOpsManager.getOps(prefix).updateView(viewIdentifier, request); + return IcebergRestUtils.ok(response); + } + + @DELETE + @Path("{view}") + @Produces(MediaType.APPLICATION_JSON) + @Timed(name = "drop-view." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) + @ResponseMetered(name = "drop-view", absolute = true) + public Response dropView( + @PathParam("prefix") String prefix, + @PathParam("namespace") String namespace, + @PathParam("view") String view) { + TableIdentifier viewIdentifier = TableIdentifier.of(RESTUtil.decodeNamespace(namespace), view); + icebergTableOpsManager.getOps(prefix).dropView(viewIdentifier); + return IcebergRestUtils.noContent(); + } + + @HEAD + @Path("{view}") + @Produces(MediaType.APPLICATION_JSON) + @Timed(name = "view-exists." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) + @ResponseMetered(name = "view-exits", absolute = true) + public Response tableExists( + @PathParam("prefix") String prefix, + @PathParam("namespace") String namespace, + @PathParam("view") String view) { + TableIdentifier tableIdentifier = TableIdentifier.of(RESTUtil.decodeNamespace(namespace), view); + if (icebergTableOpsManager.getOps(prefix).existView(tableIdentifier)) { + return IcebergRestUtils.noContent(); + } else { + return IcebergRestUtils.notExists(); + } + } + + @POST + @Path("rename") + @Produces(MediaType.APPLICATION_JSON) + @Timed(name = "rename-view." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) + @ResponseMetered(name = "rename-view", absolute = true) + public Response renameView( + @PathParam("prefix") String prefix, + @PathParam("namespace") String namespace, + RenameTableRequest request) { + icebergTableOpsManager.getOps(prefix).renameView(request); + return IcebergRestUtils.noContent(); + } +} From 7a79bf6afff342013e2db3d30d58e837dc6fbdc7 Mon Sep 17 00:00:00 2001 From: theoryxu Date: Thu, 19 Sep 2024 16:53:13 +0800 Subject: [PATCH 02/13] [#4370]feat(iceberg): support view interface for Iceberg REST server --- .../service/IcebergExceptionMapper.java | 2 + .../service/rest/IcebergViewOperations.java | 34 +- .../rest/IcebergViewRenameOperations.java | 62 ++++ .../service/rest/IcebergRestTestUtil.java | 4 + .../iceberg/service/rest/IcebergTestBase.java | 14 + .../rest/TestIcebergViewOperations.java | 311 ++++++++++++++++++ 6 files changed, 412 insertions(+), 15 deletions(-) create mode 100644 iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/rest/IcebergViewRenameOperations.java create mode 100644 iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/TestIcebergViewOperations.java diff --git a/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/IcebergExceptionMapper.java b/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/IcebergExceptionMapper.java index 95c7bf91ab9..f880f7f7a9f 100644 --- a/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/IcebergExceptionMapper.java +++ b/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/IcebergExceptionMapper.java @@ -32,6 +32,7 @@ import org.apache.iceberg.exceptions.NoSuchIcebergTableException; import org.apache.iceberg.exceptions.NoSuchNamespaceException; import org.apache.iceberg.exceptions.NoSuchTableException; +import org.apache.iceberg.exceptions.NoSuchViewException; import org.apache.iceberg.exceptions.NotAuthorizedException; import org.apache.iceberg.exceptions.ServiceUnavailableException; import org.apache.iceberg.exceptions.UnprocessableEntityException; @@ -57,6 +58,7 @@ public class IcebergExceptionMapper implements ExceptionMapper { .put(NoSuchTableException.class, 404) .put(NoSuchIcebergTableException.class, 404) .put(UnsupportedOperationException.class, 406) + .put(NoSuchViewException.class, 404) .put(AlreadyExistsException.class, 409) .put(CommitFailedException.class, 409) .put(UnprocessableEntityException.class, 422) diff --git a/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/rest/IcebergViewOperations.java b/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/rest/IcebergViewOperations.java index 6ae03433abc..87e4c38de95 100644 --- a/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/rest/IcebergViewOperations.java +++ b/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/rest/IcebergViewOperations.java @@ -1,3 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ package org.apache.gravitino.iceberg.service.rest; import com.codahale.metrics.annotation.ResponseMetered; @@ -21,7 +39,6 @@ import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.rest.RESTUtil; import org.apache.iceberg.rest.requests.CreateViewRequest; -import org.apache.iceberg.rest.requests.RenameTableRequest; import org.apache.iceberg.rest.requests.UpdateTableRequest; import org.apache.iceberg.rest.responses.ListTablesResponse; import org.apache.iceberg.rest.responses.LoadViewResponse; @@ -117,7 +134,7 @@ public Response dropView( @Produces(MediaType.APPLICATION_JSON) @Timed(name = "view-exists." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) @ResponseMetered(name = "view-exits", absolute = true) - public Response tableExists( + public Response viewExists( @PathParam("prefix") String prefix, @PathParam("namespace") String namespace, @PathParam("view") String view) { @@ -128,17 +145,4 @@ public Response tableExists( return IcebergRestUtils.notExists(); } } - - @POST - @Path("rename") - @Produces(MediaType.APPLICATION_JSON) - @Timed(name = "rename-view." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) - @ResponseMetered(name = "rename-view", absolute = true) - public Response renameView( - @PathParam("prefix") String prefix, - @PathParam("namespace") String namespace, - RenameTableRequest request) { - icebergTableOpsManager.getOps(prefix).renameView(request); - return IcebergRestUtils.noContent(); - } } diff --git a/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/rest/IcebergViewRenameOperations.java b/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/rest/IcebergViewRenameOperations.java new file mode 100644 index 00000000000..dde23754fc3 --- /dev/null +++ b/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/rest/IcebergViewRenameOperations.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.iceberg.service.rest; + +import com.codahale.metrics.annotation.ResponseMetered; +import com.codahale.metrics.annotation.Timed; +import javax.inject.Inject; +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.Consumes; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import org.apache.gravitino.iceberg.service.IcebergRestUtils; +import org.apache.gravitino.iceberg.service.IcebergTableOpsManager; +import org.apache.gravitino.metrics.MetricNames; +import org.apache.iceberg.rest.requests.RenameTableRequest; + +@Path("/v1/{prefix:([^/]*/)?}views/rename") +@Consumes(MediaType.APPLICATION_JSON) +@Produces(MediaType.APPLICATION_JSON) +public class IcebergViewRenameOperations { + + @SuppressWarnings("UnusedVariable") + @Context + private HttpServletRequest httpRequest; + + private IcebergTableOpsManager icebergTableOpsManager; + + @Inject + public IcebergViewRenameOperations(IcebergTableOpsManager icebergTableOpsManager) { + this.icebergTableOpsManager = icebergTableOpsManager; + } + + @POST + @Produces(MediaType.APPLICATION_JSON) + @Timed(name = "rename-view." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) + @ResponseMetered(name = "rename-view", absolute = true) + public Response renameView(@PathParam("prefix") String prefix, RenameTableRequest request) { + icebergTableOpsManager.getOps(prefix).renameView(request); + return IcebergRestUtils.noContent(); + } +} diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/IcebergRestTestUtil.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/IcebergRestTestUtil.java index ab36d192414..6f25d5a9982 100644 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/IcebergRestTestUtil.java +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/IcebergRestTestUtil.java @@ -44,7 +44,11 @@ public class IcebergRestTestUtil { public static final String UPDATE_NAMESPACE_POSTFIX = "properties"; public static final String TEST_NAMESPACE_NAME = "gravitino-test"; public static final String TABLE_PATH = NAMESPACE_PATH + "/" + TEST_NAMESPACE_NAME + "/tables"; + + public static final String VIEW_PATH = NAMESPACE_PATH + "/" + TEST_NAMESPACE_NAME + "/views"; public static final String RENAME_TABLE_PATH = V_1 + "/tables/rename"; + + public static final String RENAME_VIEW_PATH = V_1 + "/views/rename"; public static final String REPORT_METRICS_POSTFIX = "metrics"; public static final boolean DEBUG_SERVER_LOG_ENABLED = true; diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/IcebergTestBase.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/IcebergTestBase.java index 7d1d80b54e3..03d9a49eb28 100644 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/IcebergTestBase.java +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/IcebergTestBase.java @@ -45,16 +45,30 @@ public Invocation.Builder getRenameTableClientBuilder() { return getIcebergClientBuilder(IcebergRestTestUtil.RENAME_TABLE_PATH, Optional.empty()); } + public Invocation.Builder getRenameViewClientBuilder() { + return getIcebergClientBuilder(IcebergRestTestUtil.RENAME_VIEW_PATH, Optional.empty()); + } + public Invocation.Builder getTableClientBuilder() { return getTableClientBuilder(Optional.empty()); } + public Invocation.Builder getViewClientBuilder() { + return getViewClientBuilder(Optional.empty()); + } + public Invocation.Builder getTableClientBuilder(Optional name) { String path = Joiner.on("/").skipNulls().join(IcebergRestTestUtil.TABLE_PATH, name.orElseGet(() -> null)); return getIcebergClientBuilder(path, Optional.empty()); } + public Invocation.Builder getViewClientBuilder(Optional name) { + String path = + Joiner.on("/").skipNulls().join(IcebergRestTestUtil.VIEW_PATH, name.orElseGet(() -> null)); + return getIcebergClientBuilder(path, Optional.empty()); + } + public Invocation.Builder getReportMetricsClientBuilder(String name) { String path = Joiner.on("/") diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/TestIcebergViewOperations.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/TestIcebergViewOperations.java new file mode 100644 index 00000000000..b3f4f302d2c --- /dev/null +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/TestIcebergViewOperations.java @@ -0,0 +1,311 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.iceberg.service.rest; + +import com.google.common.collect.ImmutableSet; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import javax.ws.rs.client.Entity; +import javax.ws.rs.core.Application; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import org.apache.iceberg.Schema; +import org.apache.iceberg.UpdateRequirements; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.rest.requests.CreateViewRequest; +import org.apache.iceberg.rest.requests.ImmutableCreateViewRequest; +import org.apache.iceberg.rest.requests.RenameTableRequest; +import org.apache.iceberg.rest.requests.UpdateTableRequest; +import org.apache.iceberg.rest.responses.ListTablesResponse; +import org.apache.iceberg.rest.responses.LoadViewResponse; +import org.apache.iceberg.types.Types; +import org.apache.iceberg.view.ImmutableSQLViewRepresentation; +import org.apache.iceberg.view.ImmutableViewVersion; +import org.apache.iceberg.view.ViewMetadata; +import org.glassfish.jersey.server.ResourceConfig; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +public class TestIcebergViewOperations extends TestIcebergNamespaceOperations { + private static final Schema viewSchema = + new Schema(Types.NestedField.of(1, false, "foo_string", Types.StringType.get())); + + private static final Schema newViewSchema = + new Schema(Types.NestedField.of(2, false, "foo_string1", Types.StringType.get())); + + private static final String VIEW_QUERY = "select 1"; + + @Override + protected Application configure() { + ResourceConfig resourceConfig = + IcebergRestTestUtil.getIcebergResourceConfig(IcebergViewOperations.class); + // create namespace before each view test + resourceConfig.register(IcebergNamespaceOperations.class); + resourceConfig.register(IcebergViewRenameOperations.class); + + return resourceConfig; + } + + private Response doCreateView(String name) { + CreateViewRequest createViewRequest = + ImmutableCreateViewRequest.builder() + .name(name) + .schema(viewSchema) + .viewVersion( + ImmutableViewVersion.builder() + .versionId(1) + .timestampMillis(System.currentTimeMillis()) + .schemaId(1) + .defaultNamespace(Namespace.of(IcebergRestTestUtil.TEST_NAMESPACE_NAME)) + .addRepresentations( + ImmutableSQLViewRepresentation.builder() + .sql(VIEW_QUERY) + .dialect("spark") + .build()) + .build()) + .build(); + return getViewClientBuilder() + .post(Entity.entity(createViewRequest, MediaType.APPLICATION_JSON_TYPE)); + } + + private Response doLoadView(String name) { + return getViewClientBuilder(Optional.of(name)).get(); + } + + private void verifyLoadViewSucc(String name) { + Response response = doLoadView(name); + Assertions.assertEquals(Response.Status.OK.getStatusCode(), response.getStatus()); + + LoadViewResponse loadViewResponse = response.readEntity(LoadViewResponse.class); + Assertions.assertEquals(viewSchema.columns(), loadViewResponse.metadata().schema().columns()); + } + + private void verifyCreateViewFail(String name, int status) { + Response response = doCreateView(name); + Assertions.assertEquals(status, response.getStatus()); + } + + private void verifyCreateViewSucc(String name) { + Response response = doCreateView(name); + Assertions.assertEquals(Response.Status.OK.getStatusCode(), response.getStatus()); + LoadViewResponse loadViewResponse = response.readEntity(LoadViewResponse.class); + Schema schema = loadViewResponse.metadata().schema(); + Assertions.assertEquals(schema.columns(), viewSchema.columns()); + } + + private void verifyLoadViewFail(String name, int status) { + Response response = doLoadView(name); + Assertions.assertEquals(status, response.getStatus()); + } + + private void verifyReplaceSucc(String name, ViewMetadata base) { + Response response = doReplaceView(name, base); + Assertions.assertEquals(Response.Status.OK.getStatusCode(), response.getStatus()); + LoadViewResponse loadViewResponse = response.readEntity(LoadViewResponse.class); + Assertions.assertEquals( + newViewSchema.columns(), loadViewResponse.metadata().schema().columns()); + } + + private Response doReplaceView(String name, ViewMetadata base) { + ViewMetadata.Builder builder = + ViewMetadata.buildFrom(base).setCurrentVersion(base.currentVersion(), newViewSchema); + ViewMetadata replacement = builder.build(); + UpdateTableRequest updateTableRequest = + UpdateTableRequest.create( + null, + UpdateRequirements.forReplaceView(base, replacement.changes()), + replacement.changes()); + return getViewClientBuilder(Optional.of(name)) + .post(Entity.entity(updateTableRequest, MediaType.APPLICATION_JSON_TYPE)); + } + + private ViewMetadata getViewMeta(String viewName) { + Response response = doLoadView(viewName); + LoadViewResponse loadViewResponse = response.readEntity(LoadViewResponse.class); + return loadViewResponse.metadata(); + } + + private void verifyUpdateViewFail(String name, int status, ViewMetadata base) { + Response response = doReplaceView(name, base); + Assertions.assertEquals(status, response.getStatus()); + } + + private void verifyDropViewSucc(String name) { + Response response = doDropView(name); + Assertions.assertEquals(Response.Status.NO_CONTENT.getStatusCode(), response.getStatus()); + } + + private Response doDropView(String name) { + return getViewClientBuilder(Optional.of(name)).delete(); + } + + private void verifyDropViewFail(String name, int status) { + Response response = doDropView(name); + Assertions.assertEquals(status, response.getStatus()); + } + + private void verifyViewExistsStatusCode(String name, int status) { + Response response = doViewExists(name); + Assertions.assertEquals(status, response.getStatus()); + } + + private Response doViewExists(String name) { + return getViewClientBuilder(Optional.of(name)).head(); + } + + private void verifyListViewFail(int status) { + Response response = doListView(); + Assertions.assertEquals(status, response.getStatus()); + } + + private Response doListView() { + return getViewClientBuilder().get(); + } + + private void verifyLisViewSucc(Set expectedTableNames) { + Response response = doListView(); + Assertions.assertEquals(Response.Status.OK.getStatusCode(), response.getStatus()); + ListTablesResponse listTablesResponse = response.readEntity(ListTablesResponse.class); + Set tableNames = + listTablesResponse.identifiers().stream() + .map(identifier -> identifier.name()) + .collect(Collectors.toSet()); + Assertions.assertEquals(expectedTableNames, tableNames); + } + + private void verifyRenameViewFail(String source, String dest, int status) { + Response response = doRenameView(source, dest); + Assertions.assertEquals(status, response.getStatus()); + } + + private Response doRenameView(String source, String dest) { + RenameTableRequest renameTableRequest = + RenameTableRequest.builder() + .withSource( + TableIdentifier.of(Namespace.of(IcebergRestTestUtil.TEST_NAMESPACE_NAME), source)) + .withDestination( + TableIdentifier.of(Namespace.of(IcebergRestTestUtil.TEST_NAMESPACE_NAME), dest)) + .build(); + return getRenameViewClientBuilder() + .post(Entity.entity(renameTableRequest, MediaType.APPLICATION_JSON_TYPE)); + } + + private void verifyRenameViewSucc(String source, String dest) { + Response response = doRenameView(source, dest); + Assertions.assertEquals(Response.Status.NO_CONTENT.getStatusCode(), response.getStatus()); + } + + @ParameterizedTest + @ValueSource(strings = {"", IcebergRestTestUtil.PREFIX}) + void testListTables(String prefix) { + setUrlPathWithPrefix(prefix); + verifyListViewFail(404); + + verifyCreateNamespaceSucc(IcebergRestTestUtil.TEST_NAMESPACE_NAME); + verifyCreateViewSucc("list_foo1"); + verifyCreateViewSucc("list_foo2"); + verifyLisViewSucc(ImmutableSet.of("list_foo1", "list_foo2")); + } + + @Test + void testCreateView() { + verifyCreateViewFail("create_foo1", 404); + + verifyCreateNamespaceSucc(IcebergRestTestUtil.TEST_NAMESPACE_NAME); + + verifyCreateViewSucc("create_foo1"); + + verifyCreateViewFail("create_foo1", 409); + verifyCreateViewFail("", 400); + } + + @Test + void testLoadView() { + verifyLoadViewFail("load_foo1", 404); + + verifyCreateNamespaceSucc(IcebergRestTestUtil.TEST_NAMESPACE_NAME); + verifyCreateViewSucc("load_foo1"); + verifyLoadViewSucc("load_foo1"); + + verifyLoadViewFail("load_foo2", 404); + } + + @Test + void testReplaceView() { + verifyCreateNamespaceSucc(IcebergRestTestUtil.TEST_NAMESPACE_NAME); + verifyCreateViewSucc("replace_foo1"); + ViewMetadata metadata = getViewMeta("replace_foo1"); + verifyReplaceSucc("replace_foo1", metadata); + + verifyDropViewSucc("replace_foo1"); + verifyUpdateViewFail("replace_foo1", 404, metadata); + + verifyDropNamespaceSucc(IcebergRestTestUtil.TEST_NAMESPACE_NAME); + verifyUpdateViewFail("replace_foo1", 404, metadata); + } + + @Test + void testDropView() { + verifyDropViewFail("drop_foo1", 404); + verifyCreateNamespaceSucc(IcebergRestTestUtil.TEST_NAMESPACE_NAME); + verifyDropViewFail("drop_foo1", 404); + + verifyCreateViewSucc("drop_foo1"); + verifyDropViewSucc("drop_foo1"); + verifyLoadViewFail("drop_foo1", 404); + } + + @Test + void testViewExits() { + verifyViewExistsStatusCode("exists_foo2", 404); + verifyCreateNamespaceSucc(IcebergRestTestUtil.TEST_NAMESPACE_NAME); + verifyViewExistsStatusCode("exists_foo2", 404); + + verifyCreateViewSucc("exists_foo1"); + verifyViewExistsStatusCode("exists_foo1", 204); + verifyLoadViewSucc("exists_foo1"); + } + + @ParameterizedTest + @ValueSource(strings = {"", IcebergRestTestUtil.PREFIX}) + void testRenameTable(String prefix) { + setUrlPathWithPrefix(prefix); + // namespace not exits + verifyRenameViewFail("rename_foo1", "rename_foo3", 404); + + verifyCreateNamespaceSucc(IcebergRestTestUtil.TEST_NAMESPACE_NAME); + verifyCreateViewSucc("rename_foo1"); + // rename + verifyRenameViewSucc("rename_foo1", "rename_foo2"); + verifyLoadViewFail("rename_foo1", 404); + verifyLoadViewSucc("rename_foo2"); + + // source view not exists + verifyRenameViewFail("rename_foo1", "rename_foo3", 404); + + // dest view exists + verifyCreateViewSucc("rename_foo3"); + verifyRenameViewFail("rename_foo2", "rename_foo3", 409); + } +} From 6676adc1ae78133a55d2fa6310d3f489168ddc47 Mon Sep 17 00:00:00 2001 From: theoryxu Date: Thu, 19 Sep 2024 17:02:58 +0800 Subject: [PATCH 03/13] [#4370]feat(iceberg): support view interface for Iceberg REST server --- .../service/rest/IcebergViewOperations.java | 21 ++++++++++--------- .../rest/IcebergViewRenameOperations.java | 10 ++++----- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/rest/IcebergViewOperations.java b/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/rest/IcebergViewOperations.java index 87e4c38de95..3e46257e22b 100644 --- a/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/rest/IcebergViewOperations.java +++ b/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/rest/IcebergViewOperations.java @@ -33,8 +33,8 @@ import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; +import org.apache.gravitino.iceberg.service.IcebergCatalogWrapperManager; import org.apache.gravitino.iceberg.service.IcebergRestUtils; -import org.apache.gravitino.iceberg.service.IcebergTableOpsManager; import org.apache.gravitino.metrics.MetricNames; import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.rest.RESTUtil; @@ -48,15 +48,15 @@ @Produces(MediaType.APPLICATION_JSON) public class IcebergViewOperations { - private IcebergTableOpsManager icebergTableOpsManager; + private IcebergCatalogWrapperManager icebergCatalogWrapperManager; @SuppressWarnings("UnusedVariable") @Context private HttpServletRequest httpRequest; @Inject - public IcebergViewOperations(IcebergTableOpsManager icebergTableOpsManager) { - this.icebergTableOpsManager = icebergTableOpsManager; + public IcebergViewOperations(IcebergCatalogWrapperManager icebergCatalogWrapperManager) { + this.icebergCatalogWrapperManager = icebergCatalogWrapperManager; } @GET @@ -66,7 +66,7 @@ public IcebergViewOperations(IcebergTableOpsManager icebergTableOpsManager) { public Response listView( @PathParam("prefix") String prefix, @PathParam("namespace") String namespace) { ListTablesResponse response = - icebergTableOpsManager.getOps(prefix).listView(RESTUtil.decodeNamespace(namespace)); + icebergCatalogWrapperManager.getOps(prefix).listView(RESTUtil.decodeNamespace(namespace)); return IcebergRestUtils.ok(response); } @@ -79,7 +79,7 @@ public Response createView( @PathParam("namespace") String namespace, CreateViewRequest request) { LoadViewResponse response = - icebergTableOpsManager + icebergCatalogWrapperManager .getOps(prefix) .createView(RESTUtil.decodeNamespace(namespace), request); return IcebergRestUtils.ok(response); @@ -95,7 +95,8 @@ public Response loadView( @PathParam("namespace") String namespace, @PathParam("view") String view) { TableIdentifier viewIdentifier = TableIdentifier.of(RESTUtil.decodeNamespace(namespace), view); - LoadViewResponse response = icebergTableOpsManager.getOps(prefix).loadView(viewIdentifier); + LoadViewResponse response = + icebergCatalogWrapperManager.getOps(prefix).loadView(viewIdentifier); return IcebergRestUtils.ok(response); } @@ -111,7 +112,7 @@ public Response replaceView( UpdateTableRequest request) { TableIdentifier viewIdentifier = TableIdentifier.of(RESTUtil.decodeNamespace(namespace), view); LoadViewResponse response = - icebergTableOpsManager.getOps(prefix).updateView(viewIdentifier, request); + icebergCatalogWrapperManager.getOps(prefix).updateView(viewIdentifier, request); return IcebergRestUtils.ok(response); } @@ -125,7 +126,7 @@ public Response dropView( @PathParam("namespace") String namespace, @PathParam("view") String view) { TableIdentifier viewIdentifier = TableIdentifier.of(RESTUtil.decodeNamespace(namespace), view); - icebergTableOpsManager.getOps(prefix).dropView(viewIdentifier); + icebergCatalogWrapperManager.getOps(prefix).dropView(viewIdentifier); return IcebergRestUtils.noContent(); } @@ -139,7 +140,7 @@ public Response viewExists( @PathParam("namespace") String namespace, @PathParam("view") String view) { TableIdentifier tableIdentifier = TableIdentifier.of(RESTUtil.decodeNamespace(namespace), view); - if (icebergTableOpsManager.getOps(prefix).existView(tableIdentifier)) { + if (icebergCatalogWrapperManager.getOps(prefix).existView(tableIdentifier)) { return IcebergRestUtils.noContent(); } else { return IcebergRestUtils.notExists(); diff --git a/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/rest/IcebergViewRenameOperations.java b/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/rest/IcebergViewRenameOperations.java index dde23754fc3..128689d33be 100644 --- a/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/rest/IcebergViewRenameOperations.java +++ b/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/rest/IcebergViewRenameOperations.java @@ -30,8 +30,8 @@ import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; +import org.apache.gravitino.iceberg.service.IcebergCatalogWrapperManager; import org.apache.gravitino.iceberg.service.IcebergRestUtils; -import org.apache.gravitino.iceberg.service.IcebergTableOpsManager; import org.apache.gravitino.metrics.MetricNames; import org.apache.iceberg.rest.requests.RenameTableRequest; @@ -44,11 +44,11 @@ public class IcebergViewRenameOperations { @Context private HttpServletRequest httpRequest; - private IcebergTableOpsManager icebergTableOpsManager; + private IcebergCatalogWrapperManager icebergCatalogWrapperManager; @Inject - public IcebergViewRenameOperations(IcebergTableOpsManager icebergTableOpsManager) { - this.icebergTableOpsManager = icebergTableOpsManager; + public IcebergViewRenameOperations(IcebergCatalogWrapperManager icebergCatalogWrapperManager) { + this.icebergCatalogWrapperManager = icebergCatalogWrapperManager; } @POST @@ -56,7 +56,7 @@ public IcebergViewRenameOperations(IcebergTableOpsManager icebergTableOpsManager @Timed(name = "rename-view." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) @ResponseMetered(name = "rename-view", absolute = true) public Response renameView(@PathParam("prefix") String prefix, RenameTableRequest request) { - icebergTableOpsManager.getOps(prefix).renameView(request); + icebergCatalogWrapperManager.getOps(prefix).renameView(request); return IcebergRestUtils.noContent(); } } From e05c79e1ad20e68254b3d8d369c5f04f97d5003a Mon Sep 17 00:00:00 2001 From: theoryxu Date: Thu, 26 Sep 2024 15:57:21 +0800 Subject: [PATCH 04/13] [#4370]feat(iceberg): support view interface for Iceberg REST server --- .../rest/TestIcebergViewOperations.java | 254 +++++++++--------- 1 file changed, 127 insertions(+), 127 deletions(-) diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/TestIcebergViewOperations.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/TestIcebergViewOperations.java index b3f4f302d2c..121a61a7c96 100644 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/TestIcebergViewOperations.java +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/TestIcebergViewOperations.java @@ -67,26 +67,119 @@ protected Application configure() { return resourceConfig; } + @ParameterizedTest + @ValueSource(strings = {"", IcebergRestTestUtil.PREFIX}) + void testListTables(String prefix) { + setUrlPathWithPrefix(prefix); + verifyListViewFail(404); + + verifyCreateNamespaceSucc(IcebergRestTestUtil.TEST_NAMESPACE_NAME); + verifyCreateViewSucc("list_foo1"); + verifyCreateViewSucc("list_foo2"); + verifyLisViewSucc(ImmutableSet.of("list_foo1", "list_foo2")); + } + + @Test + void testCreateView() { + verifyCreateViewFail("create_foo1", 404); + + verifyCreateNamespaceSucc(IcebergRestTestUtil.TEST_NAMESPACE_NAME); + + verifyCreateViewSucc("create_foo1"); + + verifyCreateViewFail("create_foo1", 409); + verifyCreateViewFail("", 400); + } + + @Test + void testLoadView() { + verifyLoadViewFail("load_foo1", 404); + + verifyCreateNamespaceSucc(IcebergRestTestUtil.TEST_NAMESPACE_NAME); + verifyCreateViewSucc("load_foo1"); + verifyLoadViewSucc("load_foo1"); + + verifyLoadViewFail("load_foo2", 404); + } + + @Test + void testReplaceView() { + verifyCreateNamespaceSucc(IcebergRestTestUtil.TEST_NAMESPACE_NAME); + verifyCreateViewSucc("replace_foo1"); + ViewMetadata metadata = getViewMeta("replace_foo1"); + verifyReplaceSucc("replace_foo1", metadata); + + verifyDropViewSucc("replace_foo1"); + verifyUpdateViewFail("replace_foo1", 404, metadata); + + verifyDropNamespaceSucc(IcebergRestTestUtil.TEST_NAMESPACE_NAME); + verifyUpdateViewFail("replace_foo1", 404, metadata); + } + + @Test + void testDropView() { + verifyDropViewFail("drop_foo1", 404); + verifyCreateNamespaceSucc(IcebergRestTestUtil.TEST_NAMESPACE_NAME); + verifyDropViewFail("drop_foo1", 404); + + verifyCreateViewSucc("drop_foo1"); + verifyDropViewSucc("drop_foo1"); + verifyLoadViewFail("drop_foo1", 404); + } + + @Test + void testViewExits() { + verifyViewExistsStatusCode("exists_foo2", 404); + verifyCreateNamespaceSucc(IcebergRestTestUtil.TEST_NAMESPACE_NAME); + verifyViewExistsStatusCode("exists_foo2", 404); + + verifyCreateViewSucc("exists_foo1"); + verifyViewExistsStatusCode("exists_foo1", 204); + verifyLoadViewSucc("exists_foo1"); + } + + @ParameterizedTest + @ValueSource(strings = {"", IcebergRestTestUtil.PREFIX}) + void testRenameTable(String prefix) { + setUrlPathWithPrefix(prefix); + // namespace not exits + verifyRenameViewFail("rename_foo1", "rename_foo3", 404); + + verifyCreateNamespaceSucc(IcebergRestTestUtil.TEST_NAMESPACE_NAME); + verifyCreateViewSucc("rename_foo1"); + // rename + verifyRenameViewSucc("rename_foo1", "rename_foo2"); + verifyLoadViewFail("rename_foo1", 404); + verifyLoadViewSucc("rename_foo2"); + + // source view not exists + verifyRenameViewFail("rename_foo1", "rename_foo3", 404); + + // dest view exists + verifyCreateViewSucc("rename_foo3"); + verifyRenameViewFail("rename_foo2", "rename_foo3", 409); + } + private Response doCreateView(String name) { CreateViewRequest createViewRequest = - ImmutableCreateViewRequest.builder() - .name(name) - .schema(viewSchema) - .viewVersion( - ImmutableViewVersion.builder() - .versionId(1) - .timestampMillis(System.currentTimeMillis()) - .schemaId(1) - .defaultNamespace(Namespace.of(IcebergRestTestUtil.TEST_NAMESPACE_NAME)) - .addRepresentations( - ImmutableSQLViewRepresentation.builder() - .sql(VIEW_QUERY) - .dialect("spark") - .build()) - .build()) - .build(); + ImmutableCreateViewRequest.builder() + .name(name) + .schema(viewSchema) + .viewVersion( + ImmutableViewVersion.builder() + .versionId(1) + .timestampMillis(System.currentTimeMillis()) + .schemaId(1) + .defaultNamespace(Namespace.of(IcebergRestTestUtil.TEST_NAMESPACE_NAME)) + .addRepresentations( + ImmutableSQLViewRepresentation.builder() + .sql(VIEW_QUERY) + .dialect("spark") + .build()) + .build()) + .build(); return getViewClientBuilder() - .post(Entity.entity(createViewRequest, MediaType.APPLICATION_JSON_TYPE)); + .post(Entity.entity(createViewRequest, MediaType.APPLICATION_JSON_TYPE)); } private Response doLoadView(String name) { @@ -124,20 +217,20 @@ private void verifyReplaceSucc(String name, ViewMetadata base) { Assertions.assertEquals(Response.Status.OK.getStatusCode(), response.getStatus()); LoadViewResponse loadViewResponse = response.readEntity(LoadViewResponse.class); Assertions.assertEquals( - newViewSchema.columns(), loadViewResponse.metadata().schema().columns()); + newViewSchema.columns(), loadViewResponse.metadata().schema().columns()); } private Response doReplaceView(String name, ViewMetadata base) { ViewMetadata.Builder builder = - ViewMetadata.buildFrom(base).setCurrentVersion(base.currentVersion(), newViewSchema); + ViewMetadata.buildFrom(base).setCurrentVersion(base.currentVersion(), newViewSchema); ViewMetadata replacement = builder.build(); UpdateTableRequest updateTableRequest = - UpdateTableRequest.create( - null, - UpdateRequirements.forReplaceView(base, replacement.changes()), - replacement.changes()); + UpdateTableRequest.create( + null, + UpdateRequirements.forReplaceView(base, replacement.changes()), + replacement.changes()); return getViewClientBuilder(Optional.of(name)) - .post(Entity.entity(updateTableRequest, MediaType.APPLICATION_JSON_TYPE)); + .post(Entity.entity(updateTableRequest, MediaType.APPLICATION_JSON_TYPE)); } private ViewMetadata getViewMeta(String viewName) { @@ -188,9 +281,9 @@ private void verifyLisViewSucc(Set expectedTableNames) { Assertions.assertEquals(Response.Status.OK.getStatusCode(), response.getStatus()); ListTablesResponse listTablesResponse = response.readEntity(ListTablesResponse.class); Set tableNames = - listTablesResponse.identifiers().stream() - .map(identifier -> identifier.name()) - .collect(Collectors.toSet()); + listTablesResponse.identifiers().stream() + .map(identifier -> identifier.name()) + .collect(Collectors.toSet()); Assertions.assertEquals(expectedTableNames, tableNames); } @@ -201,111 +294,18 @@ private void verifyRenameViewFail(String source, String dest, int status) { private Response doRenameView(String source, String dest) { RenameTableRequest renameTableRequest = - RenameTableRequest.builder() - .withSource( - TableIdentifier.of(Namespace.of(IcebergRestTestUtil.TEST_NAMESPACE_NAME), source)) - .withDestination( - TableIdentifier.of(Namespace.of(IcebergRestTestUtil.TEST_NAMESPACE_NAME), dest)) - .build(); + RenameTableRequest.builder() + .withSource( + TableIdentifier.of(Namespace.of(IcebergRestTestUtil.TEST_NAMESPACE_NAME), source)) + .withDestination( + TableIdentifier.of(Namespace.of(IcebergRestTestUtil.TEST_NAMESPACE_NAME), dest)) + .build(); return getRenameViewClientBuilder() - .post(Entity.entity(renameTableRequest, MediaType.APPLICATION_JSON_TYPE)); + .post(Entity.entity(renameTableRequest, MediaType.APPLICATION_JSON_TYPE)); } private void verifyRenameViewSucc(String source, String dest) { Response response = doRenameView(source, dest); Assertions.assertEquals(Response.Status.NO_CONTENT.getStatusCode(), response.getStatus()); } - - @ParameterizedTest - @ValueSource(strings = {"", IcebergRestTestUtil.PREFIX}) - void testListTables(String prefix) { - setUrlPathWithPrefix(prefix); - verifyListViewFail(404); - - verifyCreateNamespaceSucc(IcebergRestTestUtil.TEST_NAMESPACE_NAME); - verifyCreateViewSucc("list_foo1"); - verifyCreateViewSucc("list_foo2"); - verifyLisViewSucc(ImmutableSet.of("list_foo1", "list_foo2")); - } - - @Test - void testCreateView() { - verifyCreateViewFail("create_foo1", 404); - - verifyCreateNamespaceSucc(IcebergRestTestUtil.TEST_NAMESPACE_NAME); - - verifyCreateViewSucc("create_foo1"); - - verifyCreateViewFail("create_foo1", 409); - verifyCreateViewFail("", 400); - } - - @Test - void testLoadView() { - verifyLoadViewFail("load_foo1", 404); - - verifyCreateNamespaceSucc(IcebergRestTestUtil.TEST_NAMESPACE_NAME); - verifyCreateViewSucc("load_foo1"); - verifyLoadViewSucc("load_foo1"); - - verifyLoadViewFail("load_foo2", 404); - } - - @Test - void testReplaceView() { - verifyCreateNamespaceSucc(IcebergRestTestUtil.TEST_NAMESPACE_NAME); - verifyCreateViewSucc("replace_foo1"); - ViewMetadata metadata = getViewMeta("replace_foo1"); - verifyReplaceSucc("replace_foo1", metadata); - - verifyDropViewSucc("replace_foo1"); - verifyUpdateViewFail("replace_foo1", 404, metadata); - - verifyDropNamespaceSucc(IcebergRestTestUtil.TEST_NAMESPACE_NAME); - verifyUpdateViewFail("replace_foo1", 404, metadata); - } - - @Test - void testDropView() { - verifyDropViewFail("drop_foo1", 404); - verifyCreateNamespaceSucc(IcebergRestTestUtil.TEST_NAMESPACE_NAME); - verifyDropViewFail("drop_foo1", 404); - - verifyCreateViewSucc("drop_foo1"); - verifyDropViewSucc("drop_foo1"); - verifyLoadViewFail("drop_foo1", 404); - } - - @Test - void testViewExits() { - verifyViewExistsStatusCode("exists_foo2", 404); - verifyCreateNamespaceSucc(IcebergRestTestUtil.TEST_NAMESPACE_NAME); - verifyViewExistsStatusCode("exists_foo2", 404); - - verifyCreateViewSucc("exists_foo1"); - verifyViewExistsStatusCode("exists_foo1", 204); - verifyLoadViewSucc("exists_foo1"); - } - - @ParameterizedTest - @ValueSource(strings = {"", IcebergRestTestUtil.PREFIX}) - void testRenameTable(String prefix) { - setUrlPathWithPrefix(prefix); - // namespace not exits - verifyRenameViewFail("rename_foo1", "rename_foo3", 404); - - verifyCreateNamespaceSucc(IcebergRestTestUtil.TEST_NAMESPACE_NAME); - verifyCreateViewSucc("rename_foo1"); - // rename - verifyRenameViewSucc("rename_foo1", "rename_foo2"); - verifyLoadViewFail("rename_foo1", 404); - verifyLoadViewSucc("rename_foo2"); - - // source view not exists - verifyRenameViewFail("rename_foo1", "rename_foo3", 404); - - // dest view exists - verifyCreateViewSucc("rename_foo3"); - verifyRenameViewFail("rename_foo2", "rename_foo3", 409); - } } From c6f451f9d15075c0fbd4b860b3575fddf5de7dc8 Mon Sep 17 00:00:00 2001 From: theoryxu Date: Thu, 26 Sep 2024 18:18:02 +0800 Subject: [PATCH 05/13] [#4370]feat(iceberg): support view interface for Iceberg REST server --- .../test/IcebergRESTServiceBaseIT.java | 4 + .../test/IcebergRESTServiceIT.java | 133 ++++++++++++++++++ .../rest/TestIcebergViewOperations.java | 70 ++++----- 3 files changed, 172 insertions(+), 35 deletions(-) diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTServiceBaseIT.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTServiceBaseIT.java index e562e2783e4..d373106e126 100644 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTServiceBaseIT.java +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTServiceBaseIT.java @@ -175,6 +175,10 @@ protected Map getTableInfo(String tableName) { return convertToStringMap(sql("desc table extended " + tableName)); } + protected Map getViewInfo(String viewName) { + return convertToStringMap(sql("desc extended " + viewName)); + } + protected List getTableColumns(String tableName) { List objects = sql("desc table extended " + tableName); List columns = new ArrayList<>(); diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTServiceIT.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTServiceIT.java index eb196b3a444..bf1e7022b3f 100644 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTServiceIT.java +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTServiceIT.java @@ -30,6 +30,7 @@ import org.apache.spark.sql.catalyst.analysis.NamespaceAlreadyExistsException; import org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException; import org.apache.spark.sql.catalyst.analysis.NoSuchTableException; +import org.apache.spark.sql.catalyst.analysis.NoSuchViewException; import org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; @@ -557,4 +558,136 @@ void testRegisterTable() { result = convertToStringMap(sql("SELECT * FROM iceberg_rest_table_test.register_foo2")); Assertions.assertEquals(ImmutableMap.of("1", "a", "2", "b"), result); } + + @Test + void testCreateViewAndDisplayView() { + String originTableName = "iceberg_rest_table_test.create_table_for_view_1"; + String viewName = "iceberg_rest_table_test.test_create_view"; + + sql( + String.format( + "CREATE TABLE %s ( id bigint, data string, ts timestamp) USING iceberg", + originTableName)); + sql(String.format("CREATE VIEW %s AS SELECT * FROM %s", viewName, originTableName)); + + Map viewInfo = getViewInfo(viewName); + Map m = + ImmutableMap.of( + "id", "bigint", + "data", "string", + "ts", "timestamp"); + + checkMapContains(m, viewInfo); + } + + @Test + void testViewProperties() { + String originTableName = "iceberg_rest_table_test.create_table_for_view_2"; + String viewName = "iceberg_rest_table_test.test_create_view_with_properties"; + sql( + String.format( + "CREATE TABLE %s ( id bigint, data string, ts timestamp) USING iceberg", + originTableName)); + + // test create view with properties + sql( + String.format( + "CREATE VIEW %s TBLPROPERTIES ('key1' = 'val1') AS SELECT * FROM %s", + viewName, originTableName)); + + Map viewInfo = getViewInfo(viewName); + Assertions.assertTrue(viewInfo.getOrDefault("View Properties", "").contains("'key1' = 'val1'")); + Assertions.assertFalse( + viewInfo.getOrDefault("View Properties", "").contains("'key2' = 'val2'")); + + // test set properties + sql( + String.format( + "ALTER VIEW %s SET TBLPROPERTIES ('key1' = 'val1', 'key2' = 'val2')", viewName)); + + viewInfo = getViewInfo(viewName); + Assertions.assertTrue(viewInfo.getOrDefault("View Properties", "").contains("'key1' = 'val1'")); + Assertions.assertTrue(viewInfo.getOrDefault("View Properties", "").contains("'key2' = 'val2'")); + + // test unset properties + sql(String.format("ALTER VIEW %s UNSET TBLPROPERTIES ('key1', 'key2')", viewName)); + + viewInfo = getViewInfo(viewName); + Assertions.assertFalse( + viewInfo.getOrDefault("View Properties", "").contains("'key1' = 'val1'")); + Assertions.assertFalse( + viewInfo.getOrDefault("View Properties", "").contains("'key2' = 'val2'")); + } + + @Test + void testDropView() { + String originTableName = "iceberg_rest_table_test.create_table_for_view_3"; + String viewName = "iceberg_rest_table_test.test_drop_view"; + + sql( + String.format( + "CREATE TABLE %s ( id bigint, data string, ts timestamp) USING iceberg", + originTableName)); + sql(String.format("CREATE VIEW %s AS SELECT * FROM %s", viewName, originTableName)); + sql(String.format("DROP VIEW %s", viewName)); + + Assertions.assertThrowsExactly(AnalysisException.class, () -> getViewInfo(viewName)); + Assertions.assertThrowsExactly( + NoSuchViewException.class, () -> sql(String.format("DROP VIEW %s", viewName))); + } + + @Test + void testReplaceView() { + String originTableName = "iceberg_rest_table_test.create_table_for_view_4"; + String viewName = "iceberg_rest_table_test.test_replace_view"; + + sql( + String.format( + "CREATE TABLE %s (id bigint, data string, ts timestamp) USING iceberg", + originTableName)); + sql(String.format("CREATE VIEW %s AS SELECT * FROM %s", viewName, originTableName)); + sql( + String.format( + "CREATE OR REPLACE VIEW %s (updated_id COMMENT 'updated ID') TBLPROPERTIES ('key1' = 'new_val1') AS SELECT id FROM %s", + viewName, originTableName)); + + Map viewInfo = getViewInfo(viewName); + Assertions.assertTrue( + viewInfo.getOrDefault("View Properties", "").contains("'key1' = 'new_val1'")); + Assertions.assertTrue(viewInfo.containsKey("updated_id")); + } + + @Test + void testShowAvailableViews() { + String originTableName = "iceberg_rest_table_test.create_table_for_view_5"; + String viewName1 = "iceberg_rest_table_test.show_available_views_1"; + String viewName2 = "iceberg_rest_table_test.show_available_views_2"; + + sql( + String.format( + "CREATE TABLE %s (id bigint, data string, ts timestamp) USING iceberg", + originTableName)); + sql(String.format("CREATE VIEW %s AS SELECT * FROM %s", viewName1, originTableName)); + sql(String.format("CREATE VIEW %s AS SELECT * FROM %s", viewName2, originTableName)); + + List views = sql("SHOW VIEWS IN iceberg_rest_table_test"); + Assertions.assertEquals(2, views.size()); + } + + @Test + void testShowCreateStatementView() { + String originTableName = "iceberg_rest_table_test.create_table_for_view_6"; + String viewName = "iceberg_rest_table_test.show_create_statement_view"; + + sql( + String.format( + "CREATE TABLE %s (id bigint, data string, ts timestamp) USING iceberg", + originTableName)); + sql(String.format("CREATE VIEW %s AS SELECT * FROM %s", viewName, originTableName)); + + List result = sql(String.format("SHOW CREATE TABLE %s", viewName)); + Assertions.assertEquals(1, result.size()); + Assertions.assertTrue( + Arrays.stream(result.get(0)).findFirst().orElse("").toString().contains(viewName)); + } } diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/TestIcebergViewOperations.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/TestIcebergViewOperations.java index 121a61a7c96..9ec2dc66f46 100644 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/TestIcebergViewOperations.java +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/TestIcebergViewOperations.java @@ -69,7 +69,7 @@ protected Application configure() { @ParameterizedTest @ValueSource(strings = {"", IcebergRestTestUtil.PREFIX}) - void testListTables(String prefix) { + void testListViews(String prefix) { setUrlPathWithPrefix(prefix); verifyListViewFail(404); @@ -162,24 +162,24 @@ void testRenameTable(String prefix) { private Response doCreateView(String name) { CreateViewRequest createViewRequest = - ImmutableCreateViewRequest.builder() - .name(name) - .schema(viewSchema) - .viewVersion( - ImmutableViewVersion.builder() - .versionId(1) - .timestampMillis(System.currentTimeMillis()) - .schemaId(1) - .defaultNamespace(Namespace.of(IcebergRestTestUtil.TEST_NAMESPACE_NAME)) - .addRepresentations( - ImmutableSQLViewRepresentation.builder() - .sql(VIEW_QUERY) - .dialect("spark") - .build()) - .build()) - .build(); + ImmutableCreateViewRequest.builder() + .name(name) + .schema(viewSchema) + .viewVersion( + ImmutableViewVersion.builder() + .versionId(1) + .timestampMillis(System.currentTimeMillis()) + .schemaId(1) + .defaultNamespace(Namespace.of(IcebergRestTestUtil.TEST_NAMESPACE_NAME)) + .addRepresentations( + ImmutableSQLViewRepresentation.builder() + .sql(VIEW_QUERY) + .dialect("spark") + .build()) + .build()) + .build(); return getViewClientBuilder() - .post(Entity.entity(createViewRequest, MediaType.APPLICATION_JSON_TYPE)); + .post(Entity.entity(createViewRequest, MediaType.APPLICATION_JSON_TYPE)); } private Response doLoadView(String name) { @@ -217,20 +217,20 @@ private void verifyReplaceSucc(String name, ViewMetadata base) { Assertions.assertEquals(Response.Status.OK.getStatusCode(), response.getStatus()); LoadViewResponse loadViewResponse = response.readEntity(LoadViewResponse.class); Assertions.assertEquals( - newViewSchema.columns(), loadViewResponse.metadata().schema().columns()); + newViewSchema.columns(), loadViewResponse.metadata().schema().columns()); } private Response doReplaceView(String name, ViewMetadata base) { ViewMetadata.Builder builder = - ViewMetadata.buildFrom(base).setCurrentVersion(base.currentVersion(), newViewSchema); + ViewMetadata.buildFrom(base).setCurrentVersion(base.currentVersion(), newViewSchema); ViewMetadata replacement = builder.build(); UpdateTableRequest updateTableRequest = - UpdateTableRequest.create( - null, - UpdateRequirements.forReplaceView(base, replacement.changes()), - replacement.changes()); + UpdateTableRequest.create( + null, + UpdateRequirements.forReplaceView(base, replacement.changes()), + replacement.changes()); return getViewClientBuilder(Optional.of(name)) - .post(Entity.entity(updateTableRequest, MediaType.APPLICATION_JSON_TYPE)); + .post(Entity.entity(updateTableRequest, MediaType.APPLICATION_JSON_TYPE)); } private ViewMetadata getViewMeta(String viewName) { @@ -281,9 +281,9 @@ private void verifyLisViewSucc(Set expectedTableNames) { Assertions.assertEquals(Response.Status.OK.getStatusCode(), response.getStatus()); ListTablesResponse listTablesResponse = response.readEntity(ListTablesResponse.class); Set tableNames = - listTablesResponse.identifiers().stream() - .map(identifier -> identifier.name()) - .collect(Collectors.toSet()); + listTablesResponse.identifiers().stream() + .map(identifier -> identifier.name()) + .collect(Collectors.toSet()); Assertions.assertEquals(expectedTableNames, tableNames); } @@ -294,14 +294,14 @@ private void verifyRenameViewFail(String source, String dest, int status) { private Response doRenameView(String source, String dest) { RenameTableRequest renameTableRequest = - RenameTableRequest.builder() - .withSource( - TableIdentifier.of(Namespace.of(IcebergRestTestUtil.TEST_NAMESPACE_NAME), source)) - .withDestination( - TableIdentifier.of(Namespace.of(IcebergRestTestUtil.TEST_NAMESPACE_NAME), dest)) - .build(); + RenameTableRequest.builder() + .withSource( + TableIdentifier.of(Namespace.of(IcebergRestTestUtil.TEST_NAMESPACE_NAME), source)) + .withDestination( + TableIdentifier.of(Namespace.of(IcebergRestTestUtil.TEST_NAMESPACE_NAME), dest)) + .build(); return getRenameViewClientBuilder() - .post(Entity.entity(renameTableRequest, MediaType.APPLICATION_JSON_TYPE)); + .post(Entity.entity(renameTableRequest, MediaType.APPLICATION_JSON_TYPE)); } private void verifyRenameViewSucc(String source, String dest) { From d5481b0355e66753bfc4a36c3c4e831a47ec4c4a Mon Sep 17 00:00:00 2001 From: theoryxu Date: Thu, 26 Sep 2024 19:38:30 +0800 Subject: [PATCH 06/13] [#4370]feat(iceberg): support view interface for Iceberg REST server --- .../iceberg/integration/test/IcebergRESTServiceBaseIT.java | 5 +++++ .../iceberg/integration/test/IcebergRESTServiceIT.java | 6 ++++++ 2 files changed, 11 insertions(+) diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTServiceBaseIT.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTServiceBaseIT.java index d373106e126..3f63906a2da 100644 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTServiceBaseIT.java +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTServiceBaseIT.java @@ -76,6 +76,11 @@ boolean catalogTypeNotMemory() { return !catalogType.equals(IcebergCatalogBackend.MEMORY); } + boolean catalogTypeNotHive() { + return !catalogType.equals(IcebergCatalogBackend.HIVE); + } + + abstract void initEnv(); abstract Map getCatalogConfig(); diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTServiceIT.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTServiceIT.java index bf1e7022b3f..822ee7064f8 100644 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTServiceIT.java +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTServiceIT.java @@ -560,6 +560,7 @@ void testRegisterTable() { } @Test + @EnabledIf("catalogTypeNotHive") void testCreateViewAndDisplayView() { String originTableName = "iceberg_rest_table_test.create_table_for_view_1"; String viewName = "iceberg_rest_table_test.test_create_view"; @@ -581,6 +582,7 @@ void testCreateViewAndDisplayView() { } @Test + @EnabledIf("catalogTypeNotHive") void testViewProperties() { String originTableName = "iceberg_rest_table_test.create_table_for_view_2"; String viewName = "iceberg_rest_table_test.test_create_view_with_properties"; @@ -620,6 +622,7 @@ void testViewProperties() { } @Test + @EnabledIf("catalogTypeNotHive") void testDropView() { String originTableName = "iceberg_rest_table_test.create_table_for_view_3"; String viewName = "iceberg_rest_table_test.test_drop_view"; @@ -637,6 +640,7 @@ void testDropView() { } @Test + @EnabledIf("catalogTypeNotHive") void testReplaceView() { String originTableName = "iceberg_rest_table_test.create_table_for_view_4"; String viewName = "iceberg_rest_table_test.test_replace_view"; @@ -658,6 +662,7 @@ void testReplaceView() { } @Test + @EnabledIf("catalogTypeNotHive") void testShowAvailableViews() { String originTableName = "iceberg_rest_table_test.create_table_for_view_5"; String viewName1 = "iceberg_rest_table_test.show_available_views_1"; @@ -675,6 +680,7 @@ void testShowAvailableViews() { } @Test + @EnabledIf("catalogTypeNotHive") void testShowCreateStatementView() { String originTableName = "iceberg_rest_table_test.create_table_for_view_6"; String viewName = "iceberg_rest_table_test.show_create_statement_view"; From daa29453d0c31b317924ab065570c028d576e2cf Mon Sep 17 00:00:00 2001 From: theoryxu Date: Fri, 27 Sep 2024 10:14:39 +0800 Subject: [PATCH 07/13] [#4370]feat(iceberg): support view interface for Iceberg REST server --- .../iceberg/integration/test/IcebergRESTServiceBaseIT.java | 1 - 1 file changed, 1 deletion(-) diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTServiceBaseIT.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTServiceBaseIT.java index 3f63906a2da..ce79997d4ff 100644 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTServiceBaseIT.java +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTServiceBaseIT.java @@ -80,7 +80,6 @@ boolean catalogTypeNotHive() { return !catalogType.equals(IcebergCatalogBackend.HIVE); } - abstract void initEnv(); abstract Map getCatalogConfig(); From 73e50fd8070c935764ea25b0f0302244bd2e52ef Mon Sep 17 00:00:00 2001 From: theoryxu Date: Mon, 30 Sep 2024 11:03:35 +0800 Subject: [PATCH 08/13] [#4370]feat(iceberg): support view interface for Iceberg REST server --- .../integration/test/IcebergRESTJdbcCatalogIT.java | 2 ++ .../integration/test/IcebergRESTServiceBaseIT.java | 2 +- .../integration/test/IcebergRESTServiceIT.java | 12 ++++++------ 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTJdbcCatalogIT.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTJdbcCatalogIT.java index 1dc758a15c3..d53f8022091 100644 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTJdbcCatalogIT.java +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTJdbcCatalogIT.java @@ -68,6 +68,8 @@ public Map getCatalogConfig() { configMap.put( IcebergConfig.ICEBERG_CONFIG_PREFIX + IcebergConfig.JDBC_INIT_TABLES.getKey(), "true"); + configMap.put(IcebergConfig.ICEBERG_CONFIG_PREFIX + "jdbc.schema-version", "V1"); + configMap.put( IcebergConfig.ICEBERG_CONFIG_PREFIX + IcebergConfig.CATALOG_WAREHOUSE.getKey(), GravitinoITUtils.genRandomName( diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTServiceBaseIT.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTServiceBaseIT.java index ce79997d4ff..0ba781cabd8 100644 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTServiceBaseIT.java +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTServiceBaseIT.java @@ -76,7 +76,7 @@ boolean catalogTypeNotMemory() { return !catalogType.equals(IcebergCatalogBackend.MEMORY); } - boolean catalogTypeNotHive() { + boolean isSupportsViewCatalog() { return !catalogType.equals(IcebergCatalogBackend.HIVE); } diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTServiceIT.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTServiceIT.java index 822ee7064f8..9b4900f4d75 100644 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTServiceIT.java +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTServiceIT.java @@ -560,7 +560,7 @@ void testRegisterTable() { } @Test - @EnabledIf("catalogTypeNotHive") + @EnabledIf("isSupportsViewCatalog") void testCreateViewAndDisplayView() { String originTableName = "iceberg_rest_table_test.create_table_for_view_1"; String viewName = "iceberg_rest_table_test.test_create_view"; @@ -582,7 +582,7 @@ void testCreateViewAndDisplayView() { } @Test - @EnabledIf("catalogTypeNotHive") + @EnabledIf("isSupportsViewCatalog") void testViewProperties() { String originTableName = "iceberg_rest_table_test.create_table_for_view_2"; String viewName = "iceberg_rest_table_test.test_create_view_with_properties"; @@ -622,7 +622,7 @@ void testViewProperties() { } @Test - @EnabledIf("catalogTypeNotHive") + @EnabledIf("isSupportsViewCatalog") void testDropView() { String originTableName = "iceberg_rest_table_test.create_table_for_view_3"; String viewName = "iceberg_rest_table_test.test_drop_view"; @@ -640,7 +640,7 @@ void testDropView() { } @Test - @EnabledIf("catalogTypeNotHive") + @EnabledIf("isSupportsViewCatalog") void testReplaceView() { String originTableName = "iceberg_rest_table_test.create_table_for_view_4"; String viewName = "iceberg_rest_table_test.test_replace_view"; @@ -662,7 +662,7 @@ void testReplaceView() { } @Test - @EnabledIf("catalogTypeNotHive") + @EnabledIf("isSupportsViewCatalog") void testShowAvailableViews() { String originTableName = "iceberg_rest_table_test.create_table_for_view_5"; String viewName1 = "iceberg_rest_table_test.show_available_views_1"; @@ -680,7 +680,7 @@ void testShowAvailableViews() { } @Test - @EnabledIf("catalogTypeNotHive") + @EnabledIf("isSupportsViewCatalog") void testShowCreateStatementView() { String originTableName = "iceberg_rest_table_test.create_table_for_view_6"; String viewName = "iceberg_rest_table_test.show_create_statement_view"; From c24f35a26d80dd9de58fbb2ba9b6652184310485 Mon Sep 17 00:00:00 2001 From: theoryxu Date: Tue, 8 Oct 2024 11:44:04 +0800 Subject: [PATCH 09/13] [#4370]feat(iceberg): support view interface for Iceberg REST server --- docs/iceberg-rest-service.md | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/docs/iceberg-rest-service.md b/docs/iceberg-rest-service.md index a5760118cc1..a285f2b0936 100644 --- a/docs/iceberg-rest-service.md +++ b/docs/iceberg-rest-service.md @@ -190,16 +190,17 @@ The Gravitino Iceberg REST catalog service uses the memory catalog backend by de #### JDBC backend configuration -| Configuration item | Description | Default value | Required | Since Version | -|---------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------|--------------------------|----------|---------------| -| `gravitino.iceberg-rest.catalog-backend` | The Catalog backend of the Gravitino Iceberg REST catalog service. Use the value **`jdbc`** for a JDBC catalog. | `memory` | Yes | 0.2.0 | -| `gravitino.iceberg-rest.uri` | The JDBC connection address, such as `jdbc:postgresql://127.0.0.1:5432` for Postgres, or `jdbc:mysql://127.0.0.1:3306/` for mysql. | (none) | Yes | 0.2.0 | -| `gravitino.iceberg-rest.warehouse ` | The warehouse directory of JDBC catalog. Set the HDFS prefix if using HDFS, such as `hdfs://127.0.0.1:9000/user/hive/warehouse-jdbc` | (none) | Yes | 0.2.0 | -| `gravitino.iceberg-rest.catalog-backend-name` | The catalog name passed to underlying Iceberg catalog backend. Catalog name in JDBC backend is used to isolate namespace and tables. | `jdbc` for JDBC backend | No | 0.5.2 | -| `gravitino.iceberg-rest.jdbc.user` | The username of the JDBC connection. | (none) | Yes | 0.2.0 | -| `gravitino.iceberg-rest.jdbc.password` | The password of the JDBC connection. | (none) | Yes | 0.2.0 | -| `gravitino.iceberg-rest.jdbc-initialize` | Whether to initialize the meta tables when creating the JDBC catalog. | `true` | No | 0.2.0 | -| `gravitino.iceberg-rest.jdbc-driver` | `com.mysql.jdbc.Driver` or `com.mysql.cj.jdbc.Driver` for MySQL, `org.postgresql.Driver` for PostgreSQL. | (none) | Yes | 0.3.0 | +| Configuration item | Description | Default value | Required | Since Version | +|----------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------|-------------------------|----------|---------------| +| `gravitino.iceberg-rest.catalog-backend` | The Catalog backend of the Gravitino Iceberg REST catalog service. Use the value **`jdbc`** for a JDBC catalog. | `memory` | Yes | 0.2.0 | +| `gravitino.iceberg-rest.uri` | The JDBC connection address, such as `jdbc:postgresql://127.0.0.1:5432` for Postgres, or `jdbc:mysql://127.0.0.1:3306/` for mysql. | (none) | Yes | 0.2.0 | +| `gravitino.iceberg-rest.warehouse ` | The warehouse directory of JDBC catalog. Set the HDFS prefix if using HDFS, such as `hdfs://127.0.0.1:9000/user/hive/warehouse-jdbc` | (none) | Yes | 0.2.0 | +| `gravitino.iceberg-rest.catalog-backend-name` | The catalog name passed to underlying Iceberg catalog backend. Catalog name in JDBC backend is used to isolate namespace and tables. | `jdbc` for JDBC backend | No | 0.5.2 | +| `gravitino.iceberg-rest.jdbc.user` | The username of the JDBC connection. | (none) | Yes | 0.2.0 | +| `gravitino.iceberg-rest.jdbc.password` | The password of the JDBC connection. | (none) | Yes | 0.2.0 | +| `gravitino.iceberg-rest.jdbc-initialize` | Whether to initialize the meta tables when creating the JDBC catalog. | `true` | No | 0.2.0 | +| `gravitino.iceberg-rest.jdbc-driver` | `com.mysql.jdbc.Driver` or `com.mysql.cj.jdbc.Driver` for MySQL, `org.postgresql.Driver` for PostgreSQL. | (none) | Yes | 0.3.0 | +| `gravitino.iceberg-rest.jdbc.schema-version` | JDBC catalog is initialized without view support. To auto-migrate the database's schema and enable view support, set jdbc.schema-version=V1 | (none) | NO | 0.7.0 | If you have a JDBC Iceberg catalog prior, you must set `catalog-backend-name` to keep consistent with your Jdbc Iceberg catalog name to operate the prior namespace and tables. From 99062ecbf98b8c7e715d4b1e612d1f070132459d Mon Sep 17 00:00:00 2001 From: theoryxu Date: Tue, 8 Oct 2024 12:04:10 +0800 Subject: [PATCH 10/13] [#4370]feat(iceberg): support view interface for Iceberg REST server --- docs/iceberg-rest-service.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/iceberg-rest-service.md b/docs/iceberg-rest-service.md index a285f2b0936..ba7e1467523 100644 --- a/docs/iceberg-rest-service.md +++ b/docs/iceberg-rest-service.md @@ -14,7 +14,6 @@ The Apache Gravitino Iceberg REST Server follows the [Apache Iceberg REST API sp - Supports the Apache Iceberg REST API defined in Iceberg 1.5, and supports all namespace and table interfaces. The following interfaces are not implemented yet: - token - - view - multi table transaction - pagination - Works as a catalog proxy, supporting `Hive` and `JDBC` as catalog backend. From 4bfa7f01bffc6126a672d6a9d0e66c6a63731422 Mon Sep 17 00:00:00 2001 From: theoryxu Date: Tue, 8 Oct 2024 13:07:11 +0800 Subject: [PATCH 11/13] [#4370]feat(iceberg): support view interface for Iceberg REST server --- docs/iceberg-rest-service.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/iceberg-rest-service.md b/docs/iceberg-rest-service.md index ba7e1467523..d60c3a310c1 100644 --- a/docs/iceberg-rest-service.md +++ b/docs/iceberg-rest-service.md @@ -199,7 +199,6 @@ The Gravitino Iceberg REST catalog service uses the memory catalog backend by de | `gravitino.iceberg-rest.jdbc.password` | The password of the JDBC connection. | (none) | Yes | 0.2.0 | | `gravitino.iceberg-rest.jdbc-initialize` | Whether to initialize the meta tables when creating the JDBC catalog. | `true` | No | 0.2.0 | | `gravitino.iceberg-rest.jdbc-driver` | `com.mysql.jdbc.Driver` or `com.mysql.cj.jdbc.Driver` for MySQL, `org.postgresql.Driver` for PostgreSQL. | (none) | Yes | 0.3.0 | -| `gravitino.iceberg-rest.jdbc.schema-version` | JDBC catalog is initialized without view support. To auto-migrate the database's schema and enable view support, set jdbc.schema-version=V1 | (none) | NO | 0.7.0 | If you have a JDBC Iceberg catalog prior, you must set `catalog-backend-name` to keep consistent with your Jdbc Iceberg catalog name to operate the prior namespace and tables. @@ -215,6 +214,15 @@ You must download the corresponding JDBC driver to the `iceberg-rest-server/libs If you want to use a custom Iceberg Catalog as `catalog-backend`, you can add a corresponding jar file to the classpath and load a custom Iceberg Catalog implementation by specifying the `catalog-backend-impl` property. +#### View support + +The Gravitino Iceberg REST server supports view interfaces through its JDBC backend. You can enable this by setting the `jdbc.schema-version` property. + +| Configuration item | Description | Default value | Required | Since Version | +|-------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------|-------------------------|----------|---------------| +| `gravitino.iceberg-rest.jdbc.schema-version` | JDBC catalog is initialized without view support. To auto-migrate the database's schema and enable view support, set jdbc.schema-version=V1 | (none) | NO | 0.7.0 | + + #### Multi catalog support The Gravitino Iceberg REST server supports multiple catalogs and offers a configuration-based catalog management system. From 53055a0b5d685a4afe50a8b6aedd0c38d98f8b37 Mon Sep 17 00:00:00 2001 From: theoryxu Date: Tue, 8 Oct 2024 13:09:04 +0800 Subject: [PATCH 12/13] [#4370]feat(iceberg): support view interface for Iceberg REST server --- docs/iceberg-rest-service.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/iceberg-rest-service.md b/docs/iceberg-rest-service.md index d60c3a310c1..42ffc8780e7 100644 --- a/docs/iceberg-rest-service.md +++ b/docs/iceberg-rest-service.md @@ -189,16 +189,16 @@ The Gravitino Iceberg REST catalog service uses the memory catalog backend by de #### JDBC backend configuration -| Configuration item | Description | Default value | Required | Since Version | -|----------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------|-------------------------|----------|---------------| -| `gravitino.iceberg-rest.catalog-backend` | The Catalog backend of the Gravitino Iceberg REST catalog service. Use the value **`jdbc`** for a JDBC catalog. | `memory` | Yes | 0.2.0 | -| `gravitino.iceberg-rest.uri` | The JDBC connection address, such as `jdbc:postgresql://127.0.0.1:5432` for Postgres, or `jdbc:mysql://127.0.0.1:3306/` for mysql. | (none) | Yes | 0.2.0 | -| `gravitino.iceberg-rest.warehouse ` | The warehouse directory of JDBC catalog. Set the HDFS prefix if using HDFS, such as `hdfs://127.0.0.1:9000/user/hive/warehouse-jdbc` | (none) | Yes | 0.2.0 | -| `gravitino.iceberg-rest.catalog-backend-name` | The catalog name passed to underlying Iceberg catalog backend. Catalog name in JDBC backend is used to isolate namespace and tables. | `jdbc` for JDBC backend | No | 0.5.2 | -| `gravitino.iceberg-rest.jdbc.user` | The username of the JDBC connection. | (none) | Yes | 0.2.0 | -| `gravitino.iceberg-rest.jdbc.password` | The password of the JDBC connection. | (none) | Yes | 0.2.0 | -| `gravitino.iceberg-rest.jdbc-initialize` | Whether to initialize the meta tables when creating the JDBC catalog. | `true` | No | 0.2.0 | -| `gravitino.iceberg-rest.jdbc-driver` | `com.mysql.jdbc.Driver` or `com.mysql.cj.jdbc.Driver` for MySQL, `org.postgresql.Driver` for PostgreSQL. | (none) | Yes | 0.3.0 | +| Configuration item | Description | Default value | Required | Since Version | +|---------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------|--------------------------|----------|---------------| +| `gravitino.iceberg-rest.catalog-backend` | The Catalog backend of the Gravitino Iceberg REST catalog service. Use the value **`jdbc`** for a JDBC catalog. | `memory` | Yes | 0.2.0 | +| `gravitino.iceberg-rest.uri` | The JDBC connection address, such as `jdbc:postgresql://127.0.0.1:5432` for Postgres, or `jdbc:mysql://127.0.0.1:3306/` for mysql. | (none) | Yes | 0.2.0 | +| `gravitino.iceberg-rest.warehouse ` | The warehouse directory of JDBC catalog. Set the HDFS prefix if using HDFS, such as `hdfs://127.0.0.1:9000/user/hive/warehouse-jdbc` | (none) | Yes | 0.2.0 | +| `gravitino.iceberg-rest.catalog-backend-name` | The catalog name passed to underlying Iceberg catalog backend. Catalog name in JDBC backend is used to isolate namespace and tables. | `jdbc` for JDBC backend | No | 0.5.2 | +| `gravitino.iceberg-rest.jdbc.user` | The username of the JDBC connection. | (none) | Yes | 0.2.0 | +| `gravitino.iceberg-rest.jdbc.password` | The password of the JDBC connection. | (none) | Yes | 0.2.0 | +| `gravitino.iceberg-rest.jdbc-initialize` | Whether to initialize the meta tables when creating the JDBC catalog. | `true` | No | 0.2.0 | +| `gravitino.iceberg-rest.jdbc-driver` | `com.mysql.jdbc.Driver` or `com.mysql.cj.jdbc.Driver` for MySQL, `org.postgresql.Driver` for PostgreSQL. | (none) | Yes | 0.3.0 | If you have a JDBC Iceberg catalog prior, you must set `catalog-backend-name` to keep consistent with your Jdbc Iceberg catalog name to operate the prior namespace and tables. From 6ec3b8aab29e8a7a2c8887d47682fc5bdf22f70b Mon Sep 17 00:00:00 2001 From: theoryxu Date: Tue, 8 Oct 2024 14:53:26 +0800 Subject: [PATCH 13/13] [#4370]feat(iceberg): support view interface for Iceberg REST server --- docs/iceberg-rest-service.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/iceberg-rest-service.md b/docs/iceberg-rest-service.md index 42ffc8780e7..72421f0e779 100644 --- a/docs/iceberg-rest-service.md +++ b/docs/iceberg-rest-service.md @@ -216,11 +216,11 @@ If you want to use a custom Iceberg Catalog as `catalog-backend`, you can add a #### View support -The Gravitino Iceberg REST server supports view interfaces through its JDBC backend. You can enable this by setting the `jdbc.schema-version` property. +You could access the view interface if using JDBC backend and enable `jdbc.schema-version` property. -| Configuration item | Description | Default value | Required | Since Version | -|-------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------|-------------------------|----------|---------------| -| `gravitino.iceberg-rest.jdbc.schema-version` | JDBC catalog is initialized without view support. To auto-migrate the database's schema and enable view support, set jdbc.schema-version=V1 | (none) | NO | 0.7.0 | +| Configuration item | Description | Default value | Required | Since Version | +|-------------------------------------------------|--------------------------------------------------------------------------------------------|---------------|----------|---------------| +| `gravitino.iceberg-rest.jdbc.schema-version` | The schema version of JDBC catalog backend, setting to `V1` if supporting view operations. | (none) | NO | 0.7.0 | #### Multi catalog support