Skip to content

Commit

Permalink
Add test API for catalog creation
Browse files Browse the repository at this point in the history
  • Loading branch information
mchades committed Jul 9, 2024
1 parent 178eb37 commit ce2fcb4
Show file tree
Hide file tree
Showing 38 changed files with 1,048 additions and 49 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@
* altering and dropping catalogs.
*/
@Evolving
public interface SupportsCatalogs {
public interface SupportsCatalogs extends TestOperations {

/**
* List the name of all catalogs in the metalake.
Expand Down
74 changes: 74 additions & 0 deletions api/src/main/java/com/datastrato/gravitino/TestOperations.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package com.datastrato.gravitino;

import com.google.common.base.Preconditions;
import java.util.Map;

public interface TestOperations {
/**
* Test whether a catalog can be created with the specified parameters, without actually creating
* it.
*
* @param catalogName the name of the catalog.
* @param type the type of the catalog.
* @param provider the provider of the catalog.
* @param comment the comment of the catalog.
* @param properties the properties of the catalog.
* @return The test result.
*/
TestResult testCatalogCreation(
String catalogName,
Catalog.Type type,
String provider,
String comment,
Map<String, String> properties);

interface TestResult {
TestResult VALID = new TestResultImpl(true, null);

static TestResult invalid(Exception exception) {
return new TestResultImpl(false, exception);
}

boolean valid();

Exception exception();
}

class TestResultImpl implements TestResult {
private boolean valid;
private Exception exception;

private TestResultImpl(boolean valid, Exception exception) {
Preconditions.checkArgument(
valid || exception != null, "exception is required when valid is false");
this.valid = valid;
this.exception = exception;
}

public boolean valid() {
return valid;
}

public Exception exception() {
return exception;
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package com.datastrato.gravitino.exceptions;

import com.google.errorprone.annotations.FormatMethod;
import com.google.errorprone.annotations.FormatString;

/** An exception thrown when a connect to catalog failed. */
public class ConnectionFailedException extends GravitinoRuntimeException {
/**
* Constructs a new exception with the specified detail message.
*
* @param cause the cause.
* @param errorMessageTemplate the detail message.
* @param args the arguments to the message.
*/
@FormatMethod
public ConnectionFailedException(
Throwable cause, @FormatString String errorMessageTemplate, Object... args) {
super(cause, errorMessageTemplate, args);
}

/**
* Constructs a new exception with the specified detail message.
*
* @param errorMessageTemplate the detail message.
* @param args the arguments to the message.
*/
@FormatMethod
public ConnectionFailedException(@FormatString String errorMessageTemplate, Object... args) {
super(errorMessageTemplate, args);
}
}
3 changes: 2 additions & 1 deletion build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -489,7 +489,8 @@ tasks.rat {
"clients/client-python/.pytest_cache/*",
"clients/client-python/gravitino.egg-info/*",
"clients/client-python/gravitino/utils/exceptions.py",
"clients/client-python/gravitino/utils/http_client.py"
"clients/client-python/gravitino/utils/http_client.py",
"clients/client-python/tests/unittests/htmlcov/*"
)

// Add .gitignore excludes to the Apache Rat exclusion list.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import static com.datastrato.gravitino.connector.BaseCatalog.CATALOG_BYPASS_PREFIX;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION;

import com.datastrato.gravitino.Catalog;
import com.datastrato.gravitino.Entity;
import com.datastrato.gravitino.EntityStore;
import com.datastrato.gravitino.GravitinoEnv;
Expand All @@ -29,6 +30,7 @@
import com.datastrato.gravitino.Schema;
import com.datastrato.gravitino.SchemaChange;
import com.datastrato.gravitino.StringIdentifier;
import com.datastrato.gravitino.TestOperations;
import com.datastrato.gravitino.catalog.hadoop.authentication.AuthenticationConfig;
import com.datastrato.gravitino.catalog.hadoop.authentication.kerberos.KerberosClient;
import com.datastrato.gravitino.connector.CatalogInfo;
Expand Down Expand Up @@ -72,7 +74,8 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class HadoopCatalogOperations implements CatalogOperations, SupportsSchemas, FilesetCatalog {
public class HadoopCatalogOperations
implements CatalogOperations, SupportsSchemas, FilesetCatalog, TestOperations {

private static final String SCHEMA_DOES_NOT_EXIST_MSG = "Schema %s does not exist";
private static final String FILESET_DOES_NOT_EXIST_MSG = "Fileset %s does not exist";
Expand Down Expand Up @@ -538,6 +541,26 @@ public boolean dropSchema(NameIdentifier ident, boolean cascade) throws NonEmpty
}
}

/**
* Since the Hadoop catalog was completely managed by Gravitino, always return VALID.
*
* @param catalogName the name of the catalog.
* @param type the type of the catalog.
* @param provider the provider of the catalog.
* @param comment the comment of the catalog.
* @param properties the properties of the catalog.
* @return Always return {@link TestResult#VALID}
*/
@Override
public TestResult testCatalogCreation(
String catalogName,
Catalog.Type type,
String provider,
String comment,
Map<String, String> properties) {
return TestResult.VALID;
}

@Override
public void close() throws IOException {}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.when;

import com.datastrato.gravitino.Catalog;
import com.datastrato.gravitino.Config;
import com.datastrato.gravitino.EntityStore;
import com.datastrato.gravitino.EntityStoreFactory;
Expand All @@ -46,6 +47,7 @@
import com.datastrato.gravitino.Schema;
import com.datastrato.gravitino.SchemaChange;
import com.datastrato.gravitino.StringIdentifier;
import com.datastrato.gravitino.TestOperations;
import com.datastrato.gravitino.connector.HasPropertyMetadata;
import com.datastrato.gravitino.connector.PropertiesMetadata;
import com.datastrato.gravitino.exceptions.NoSuchFilesetException;
Expand Down Expand Up @@ -680,6 +682,16 @@ public void testUpdateFilesetComment() throws IOException {
}
}

@Test
public void testTestCatalogCreation() {
HadoopCatalogOperations catalogOperations = new HadoopCatalogOperations(store);
TestOperations.TestResult testResult =
catalogOperations.testCatalogCreation(
"catalog", Catalog.Type.FILESET, "hadoop", "comment", ImmutableMap.of());
Assertions.assertTrue(testResult.valid());
Assertions.assertNull(testResult.exception());
}

private static Stream<Arguments> locationArguments() {
return Stream.of(
// Honor the catalog location
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import com.datastrato.gravitino.Schema;
import com.datastrato.gravitino.client.GravitinoMetalake;
import com.datastrato.gravitino.exceptions.FilesetAlreadyExistsException;
import com.datastrato.gravitino.exceptions.IllegalNameIdentifierException;
import com.datastrato.gravitino.exceptions.NoSuchFilesetException;
import com.datastrato.gravitino.file.Fileset;
import com.datastrato.gravitino.file.FilesetChange;
Expand Down Expand Up @@ -187,7 +188,7 @@ public void testCreateFileset() throws IOException {

// create fileset with null fileset name
Assertions.assertThrows(
IllegalArgumentException.class,
IllegalNameIdentifierException.class,
() ->
createFileset(
null,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
import com.datastrato.gravitino.Schema;
import com.datastrato.gravitino.client.GravitinoMetalake;
import com.datastrato.gravitino.exceptions.FilesetAlreadyExistsException;
import com.datastrato.gravitino.exceptions.IllegalNameIdentifierException;
import com.datastrato.gravitino.file.Fileset;
import com.datastrato.gravitino.integration.test.util.AbstractIT;
import com.datastrato.gravitino.integration.test.util.GravitinoITUtils;
Expand Down Expand Up @@ -398,7 +399,7 @@ public void testCreateFileset() throws Exception {

// create fileset with null fileset name
Assertions.assertThrows(
IllegalArgumentException.class,
IllegalNameIdentifierException.class,
() ->
createFileset(
null,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,15 +32,18 @@
import static com.datastrato.gravitino.connector.BaseCatalog.CATALOG_BYPASS_PREFIX;
import static org.apache.hadoop.hive.metastore.TableType.EXTERNAL_TABLE;

import com.datastrato.gravitino.Catalog;
import com.datastrato.gravitino.NameIdentifier;
import com.datastrato.gravitino.Namespace;
import com.datastrato.gravitino.SchemaChange;
import com.datastrato.gravitino.TestOperations;
import com.datastrato.gravitino.catalog.hive.HiveTablePropertiesMetadata.TableType;
import com.datastrato.gravitino.connector.CatalogInfo;
import com.datastrato.gravitino.connector.CatalogOperations;
import com.datastrato.gravitino.connector.HasPropertyMetadata;
import com.datastrato.gravitino.connector.ProxyPlugin;
import com.datastrato.gravitino.connector.SupportsSchemas;
import com.datastrato.gravitino.exceptions.ConnectionFailedException;
import com.datastrato.gravitino.exceptions.NoSuchCatalogException;
import com.datastrato.gravitino.exceptions.NoSuchSchemaException;
import com.datastrato.gravitino.exceptions.NoSuchTableException;
Expand Down Expand Up @@ -85,6 +88,7 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.metastore.IMetaStoreClient;
import org.apache.hadoop.hive.metastore.api.AlreadyExistsException;
import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
Expand All @@ -100,7 +104,8 @@
import org.slf4j.LoggerFactory;

/** Operations for interacting with an Apache Hive catalog in Apache Gravitino. */
public class HiveCatalogOperations implements CatalogOperations, SupportsSchemas, TableCatalog {
public class HiveCatalogOperations
implements CatalogOperations, SupportsSchemas, TableCatalog, TestOperations {

public static final Logger LOG = LoggerFactory.getLogger(HiveCatalogOperations.class);
public static final String GRAVITINO_KEYTAB_FORMAT = "keytabs/gravitino-%s-keytab";
Expand Down Expand Up @@ -1087,6 +1092,33 @@ public boolean purgeTable(NameIdentifier tableIdent) throws UnsupportedOperation
}
}

/**
* Performs `getAllDatabases` operation in Hive Metastore to test the connection.
*
* @param catalogName the name of the catalog.
* @param type the type of the catalog.
* @param provider the provider of the catalog.
* @param comment the comment of the catalog.
* @param properties the properties of the catalog.
* @return the test result.
*/
@Override
public TestResult testCatalogCreation(
String catalogName,
Catalog.Type type,
String provider,
String comment,
Map<String, String> properties) {
try {
clientPool.run(IMetaStoreClient::getAllDatabases);
} catch (Exception e) {
return TestResult.invalid(
new ConnectionFailedException(
e, "Failed to run getAllDatabases in Hive Metastore: %s", e.getMessage()));
}
return TestResult.VALID;
}

/**
* Checks if the given namespace is a valid namespace for the Hive schema.
*
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,13 +32,20 @@
import static com.datastrato.gravitino.catalog.hive.HiveCatalogPropertiesMeta.PRINCIPAL;
import static com.datastrato.gravitino.catalog.hive.TestHiveCatalog.HIVE_PROPERTIES_METADATA;
import static com.datastrato.gravitino.connector.BaseCatalog.CATALOG_BYPASS_PREFIX;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;

import com.datastrato.gravitino.Catalog;
import com.datastrato.gravitino.TestOperations;
import com.datastrato.gravitino.connector.BaseCatalog;
import com.datastrato.gravitino.connector.PropertyEntry;
import com.datastrato.gravitino.exceptions.ConnectionFailedException;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps;
import java.util.Map;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.thrift.TException;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;

Expand Down Expand Up @@ -125,4 +132,17 @@ void testPropertyOverwrite() {
Assertions.assertEquals("v2", op.hiveConf.get("a.b"));
Assertions.assertEquals("v4", op.hiveConf.get("c.d"));
}

@Test
void testTestCatalogCreation() throws TException, InterruptedException {
HiveCatalogOperations op = new HiveCatalogOperations();
op.clientPool = mock(CachedClientPool.class);
when(op.clientPool.run(any())).thenThrow(new TException("mock connection exception"));

TestOperations.TestResult result =
op.testCatalogCreation(
"catalog", Catalog.Type.RELATIONAL, "hive", "comment", ImmutableMap.of());
Assertions.assertFalse(result.valid());
Assertions.assertInstanceOf(ConnectionFailedException.class, result.exception());
}
}
Loading

0 comments on commit ce2fcb4

Please sign in to comment.