diff --git a/lib/trino-hdfs/src/main/java/io/trino/hdfs/gcs/GcsAccessTokenProvider.java b/lib/trino-hdfs/src/main/java/io/trino/hdfs/gcs/GcsAccessTokenProvider.java index 21b4877ff704a..215e6c0d4b286 100644 --- a/lib/trino-hdfs/src/main/java/io/trino/hdfs/gcs/GcsAccessTokenProvider.java +++ b/lib/trino-hdfs/src/main/java/io/trino/hdfs/gcs/GcsAccessTokenProvider.java @@ -16,8 +16,6 @@ import com.google.cloud.hadoop.util.AccessTokenProvider; import org.apache.hadoop.conf.Configuration; -import java.time.Instant; - import static com.google.common.base.Strings.nullToEmpty; import static java.util.concurrent.TimeUnit.HOURS; @@ -31,7 +29,7 @@ public class GcsAccessTokenProvider @Override public AccessToken getAccessToken() { - return new AccessToken(nullToEmpty(config.get(GCS_ACCESS_TOKEN_CONF)), Instant.now().plusMillis(EXPIRATION_TIME_MILLISECONDS)); + return new AccessToken(nullToEmpty(config.get(GCS_ACCESS_TOKEN_CONF)), EXPIRATION_TIME_MILLISECONDS); } @Override diff --git a/lib/trino-hdfs/src/main/java/io/trino/hdfs/gcs/GcsStorageFactory.java b/lib/trino-hdfs/src/main/java/io/trino/hdfs/gcs/GcsStorageFactory.java index 55384769f2592..10aad9ec5471d 100644 --- a/lib/trino-hdfs/src/main/java/io/trino/hdfs/gcs/GcsStorageFactory.java +++ b/lib/trino-hdfs/src/main/java/io/trino/hdfs/gcs/GcsStorageFactory.java @@ -13,10 +13,10 @@ */ package io.trino.hdfs.gcs; +import com.google.cloud.hadoop.repackaged.gcs.com.google.api.client.googleapis.auth.oauth2.GoogleCredential; import com.google.cloud.hadoop.repackaged.gcs.com.google.api.client.http.HttpTransport; import com.google.cloud.hadoop.repackaged.gcs.com.google.api.client.json.jackson2.JacksonFactory; import com.google.cloud.hadoop.repackaged.gcs.com.google.api.services.storage.Storage; -import com.google.cloud.hadoop.repackaged.gcs.com.google.auth.oauth2.GoogleCredentials; import com.google.cloud.hadoop.repackaged.gcs.com.google.cloud.hadoop.gcsio.GoogleCloudStorageOptions; import com.google.cloud.hadoop.repackaged.gcs.com.google.cloud.hadoop.util.HttpTransportFactory; import com.google.cloud.hadoop.repackaged.gcs.com.google.cloud.hadoop.util.RetryHttpInitializer; @@ -31,10 +31,11 @@ import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; +import java.time.Duration; import java.util.Optional; import static com.google.cloud.hadoop.fs.gcs.TrinoGoogleHadoopFileSystemConfiguration.getGcsOptionsBuilder; -import static com.google.cloud.hadoop.repackaged.gcs.com.google.cloud.hadoop.util.HadoopCredentialsConfiguration.CLOUD_PLATFORM_SCOPE; +import static com.google.cloud.hadoop.repackaged.gcs.com.google.cloud.hadoop.util.CredentialFactory.DEFAULT_SCOPES; import static com.google.common.base.Strings.nullToEmpty; import static io.trino.hdfs.gcs.GcsConfigurationProvider.GCS_OAUTH_KEY; import static io.trino.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; @@ -46,7 +47,7 @@ public class GcsStorageFactory private static final String APPLICATION_NAME = "Trino"; private final boolean useGcsAccessToken; - private final Optional jsonGoogleCredential; + private final Optional jsonGoogleCredential; @Inject public GcsStorageFactory(HiveGcsConfig hiveGcsConfig) @@ -58,12 +59,12 @@ public GcsStorageFactory(HiveGcsConfig hiveGcsConfig) String jsonKeyFilePath = hiveGcsConfig.getJsonKeyFilePath(); if (jsonKey != null) { try (InputStream inputStream = new ByteArrayInputStream(jsonKey.getBytes(UTF_8))) { - jsonGoogleCredential = Optional.of(GoogleCredentials.fromStream(inputStream).createScoped(CLOUD_PLATFORM_SCOPE)); + jsonGoogleCredential = Optional.of(GoogleCredential.fromStream(inputStream).createScoped(DEFAULT_SCOPES)); } } else if (jsonKeyFilePath != null) { try (FileInputStream inputStream = new FileInputStream(jsonKeyFilePath)) { - jsonGoogleCredential = Optional.of(GoogleCredentials.fromStream(inputStream).createScoped(CLOUD_PLATFORM_SCOPE)); + jsonGoogleCredential = Optional.of(GoogleCredential.fromStream(inputStream).createScoped(DEFAULT_SCOPES)); } } else { @@ -76,23 +77,24 @@ public Storage create(HdfsEnvironment environment, HdfsContext context, Path pat try { GoogleCloudStorageOptions gcsOptions = getGcsOptionsBuilder(environment.getConfiguration(context, path)).build(); HttpTransport httpTransport = HttpTransportFactory.createHttpTransport( + gcsOptions.getTransportType(), gcsOptions.getProxyAddress(), gcsOptions.getProxyUsername(), gcsOptions.getProxyPassword()); - GoogleCredentials credential; + GoogleCredential credential; if (useGcsAccessToken) { String accessToken = nullToEmpty(context.getIdentity().getExtraCredentials().get(GCS_OAUTH_KEY)); try (ByteArrayInputStream inputStream = new ByteArrayInputStream(accessToken.getBytes(UTF_8))) { - credential = GoogleCredentials.fromStream(inputStream).createScoped(CLOUD_PLATFORM_SCOPE); + credential = GoogleCredential.fromStream(inputStream).createScoped(DEFAULT_SCOPES); } } else { credential = jsonGoogleCredential.orElseThrow(() -> new IllegalStateException("GCS credentials not configured")); } return new Storage.Builder(httpTransport, JacksonFactory.getDefaultInstance(), new RetryHttpInitializer(credential, RetryHttpInitializerOptions.builder() - .setReadTimeout(gcsOptions.getHttpRequestReadTimeout()) + .setReadTimeout(Duration.ofMillis(gcsOptions.getHttpRequestReadTimeout())) .setMaxRequestRetries(gcsOptions.getMaxHttpRequestRetries()) - .build())) + .build())) .setApplicationName(APPLICATION_NAME) .build(); } diff --git a/lib/trino-hdfs/src/main/java/io/trino/hdfs/gcs/GoogleGcsConfigurationInitializer.java b/lib/trino-hdfs/src/main/java/io/trino/hdfs/gcs/GoogleGcsConfigurationInitializer.java index 4e979bf2e1a3e..84945a83f8eae 100644 --- a/lib/trino-hdfs/src/main/java/io/trino/hdfs/gcs/GoogleGcsConfigurationInitializer.java +++ b/lib/trino-hdfs/src/main/java/io/trino/hdfs/gcs/GoogleGcsConfigurationInitializer.java @@ -29,11 +29,9 @@ import java.util.Optional; import static com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystemConfiguration.GCS_CONFIG_PREFIX; -import static com.google.cloud.hadoop.fs.gcs.HadoopCredentialsConfiguration.SERVICE_ACCOUNT_JSON_KEYFILE_SUFFIX; -import static com.google.cloud.hadoop.repackaged.gcs.com.google.cloud.hadoop.util.HadoopCredentialsConfiguration.ACCESS_TOKEN_PROVIDER_SUFFIX; -import static com.google.cloud.hadoop.repackaged.gcs.com.google.cloud.hadoop.util.HadoopCredentialsConfiguration.AUTHENTICATION_TYPE_SUFFIX; -import static com.google.cloud.hadoop.repackaged.gcs.com.google.cloud.hadoop.util.HadoopCredentialsConfiguration.AuthenticationType.ACCESS_TOKEN_PROVIDER; -import static com.google.cloud.hadoop.repackaged.gcs.com.google.cloud.hadoop.util.HadoopCredentialsConfiguration.AuthenticationType.SERVICE_ACCOUNT_JSON_KEYFILE; +import static com.google.cloud.hadoop.repackaged.gcs.com.google.cloud.hadoop.util.HadoopCredentialConfiguration.ACCESS_TOKEN_PROVIDER_IMPL_SUFFIX; +import static com.google.cloud.hadoop.repackaged.gcs.com.google.cloud.hadoop.util.HadoopCredentialConfiguration.ENABLE_SERVICE_ACCOUNTS_SUFFIX; +import static com.google.cloud.hadoop.repackaged.gcs.com.google.cloud.hadoop.util.HadoopCredentialConfiguration.SERVICE_ACCOUNT_JSON_KEYFILE_SUFFIX; import static java.nio.file.attribute.PosixFilePermission.OWNER_READ; import static java.nio.file.attribute.PosixFilePermission.OWNER_WRITE; @@ -74,12 +72,12 @@ public void initializeConfiguration(Configuration config) if (useGcsAccessToken) { // use oauth token to authenticate with Google Cloud Storage - config.setEnum(GCS_CONFIG_PREFIX + AUTHENTICATION_TYPE_SUFFIX.getKey(), ACCESS_TOKEN_PROVIDER); - config.setClass(GCS_CONFIG_PREFIX + ACCESS_TOKEN_PROVIDER_SUFFIX.getKey(), GcsAccessTokenProvider.class, AccessTokenProvider.class); + config.setBoolean(GCS_CONFIG_PREFIX + ENABLE_SERVICE_ACCOUNTS_SUFFIX.getKey(), false); + config.setClass(GCS_CONFIG_PREFIX + ACCESS_TOKEN_PROVIDER_IMPL_SUFFIX.getKey(), GcsAccessTokenProvider.class, AccessTokenProvider.class); } else if (jsonKeyFilePath != null) { // use service account key file - config.setEnum(GCS_CONFIG_PREFIX + AUTHENTICATION_TYPE_SUFFIX.getKey(), SERVICE_ACCOUNT_JSON_KEYFILE); + config.setBoolean(GCS_CONFIG_PREFIX + ENABLE_SERVICE_ACCOUNTS_SUFFIX.getKey(), true); config.set(GCS_CONFIG_PREFIX + SERVICE_ACCOUNT_JSON_KEYFILE_SUFFIX.getKey(), jsonKeyFilePath); } } diff --git a/pom.xml b/pom.xml index 2bf89f982f83b..004fcf15986d6 100644 --- a/pom.xml +++ b/pom.xml @@ -522,7 +522,7 @@ com.google.cloud.bigdataoss gcs-connector - 3.0.0 + hadoop3-2.2.19 shaded