Update gcs-connector to 2.2.19

It partially reverts update to 3.0.0 which seems to be less supported version with some critical CVEs in the shaded jar.
trinodb · Feb 14, 2024 · 03efedf · 03efedf
1 parent c662a33
commit 03efedf
Show file tree

Hide file tree

Showing 4 changed files with 19 additions and 21 deletions.
diff --git a/lib/trino-hdfs/src/main/java/io/trino/hdfs/gcs/GcsAccessTokenProvider.java b/lib/trino-hdfs/src/main/java/io/trino/hdfs/gcs/GcsAccessTokenProvider.java
@@ -16,8 +16,6 @@
 import com.google.cloud.hadoop.util.AccessTokenProvider;
 import org.apache.hadoop.conf.Configuration;
 
-import java.time.Instant;
-
 import static com.google.common.base.Strings.nullToEmpty;
 import static java.util.concurrent.TimeUnit.HOURS;
 
@@ -31,7 +29,7 @@ public class GcsAccessTokenProvider
     @Override
     public AccessToken getAccessToken()
     {
-        return new AccessToken(nullToEmpty(config.get(GCS_ACCESS_TOKEN_CONF)), Instant.now().plusMillis(EXPIRATION_TIME_MILLISECONDS));
+        return new AccessToken(nullToEmpty(config.get(GCS_ACCESS_TOKEN_CONF)), EXPIRATION_TIME_MILLISECONDS);
     }
 
     @Override

diff --git a/lib/trino-hdfs/src/main/java/io/trino/hdfs/gcs/GcsStorageFactory.java b/lib/trino-hdfs/src/main/java/io/trino/hdfs/gcs/GcsStorageFactory.java
@@ -13,10 +13,10 @@
  */
 package io.trino.hdfs.gcs;
 
+import com.google.cloud.hadoop.repackaged.gcs.com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
 import com.google.cloud.hadoop.repackaged.gcs.com.google.api.client.http.HttpTransport;
 import com.google.cloud.hadoop.repackaged.gcs.com.google.api.client.json.jackson2.JacksonFactory;
 import com.google.cloud.hadoop.repackaged.gcs.com.google.api.services.storage.Storage;
-import com.google.cloud.hadoop.repackaged.gcs.com.google.auth.oauth2.GoogleCredentials;
 import com.google.cloud.hadoop.repackaged.gcs.com.google.cloud.hadoop.gcsio.GoogleCloudStorageOptions;
 import com.google.cloud.hadoop.repackaged.gcs.com.google.cloud.hadoop.util.HttpTransportFactory;
 import com.google.cloud.hadoop.repackaged.gcs.com.google.cloud.hadoop.util.RetryHttpInitializer;
@@ -31,10 +31,11 @@
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.time.Duration;
 import java.util.Optional;
 
 import static com.google.cloud.hadoop.fs.gcs.TrinoGoogleHadoopFileSystemConfiguration.getGcsOptionsBuilder;
-import static com.google.cloud.hadoop.repackaged.gcs.com.google.cloud.hadoop.util.HadoopCredentialsConfiguration.CLOUD_PLATFORM_SCOPE;
+import static com.google.cloud.hadoop.repackaged.gcs.com.google.cloud.hadoop.util.CredentialFactory.DEFAULT_SCOPES;
 import static com.google.common.base.Strings.nullToEmpty;
 import static io.trino.hdfs.gcs.GcsConfigurationProvider.GCS_OAUTH_KEY;
 import static io.trino.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR;
@@ -46,7 +47,7 @@ public class GcsStorageFactory
     private static final String APPLICATION_NAME = "Trino";
 
     private final boolean useGcsAccessToken;
-    private final Optional<GoogleCredentials> jsonGoogleCredential;
+    private final Optional<GoogleCredential> jsonGoogleCredential;
 
     @Inject
     public GcsStorageFactory(HiveGcsConfig hiveGcsConfig)
@@ -58,12 +59,12 @@ public GcsStorageFactory(HiveGcsConfig hiveGcsConfig)
         String jsonKeyFilePath = hiveGcsConfig.getJsonKeyFilePath();
         if (jsonKey != null) {
             try (InputStream inputStream = new ByteArrayInputStream(jsonKey.getBytes(UTF_8))) {
-                jsonGoogleCredential = Optional.of(GoogleCredentials.fromStream(inputStream).createScoped(CLOUD_PLATFORM_SCOPE));
+                jsonGoogleCredential = Optional.of(GoogleCredential.fromStream(inputStream).createScoped(DEFAULT_SCOPES));
             }
         }
         else if (jsonKeyFilePath != null) {
             try (FileInputStream inputStream = new FileInputStream(jsonKeyFilePath)) {
-                jsonGoogleCredential = Optional.of(GoogleCredentials.fromStream(inputStream).createScoped(CLOUD_PLATFORM_SCOPE));
+                jsonGoogleCredential = Optional.of(GoogleCredential.fromStream(inputStream).createScoped(DEFAULT_SCOPES));
             }
         }
         else {
@@ -76,23 +77,24 @@ public Storage create(HdfsEnvironment environment, HdfsContext context, Path pat
         try {
             GoogleCloudStorageOptions gcsOptions = getGcsOptionsBuilder(environment.getConfiguration(context, path)).build();
             HttpTransport httpTransport = HttpTransportFactory.createHttpTransport(
+                    gcsOptions.getTransportType(),
                     gcsOptions.getProxyAddress(),
                     gcsOptions.getProxyUsername(),
                     gcsOptions.getProxyPassword());
-            GoogleCredentials credential;
+            GoogleCredential credential;
             if (useGcsAccessToken) {
                 String accessToken = nullToEmpty(context.getIdentity().getExtraCredentials().get(GCS_OAUTH_KEY));
                 try (ByteArrayInputStream inputStream = new ByteArrayInputStream(accessToken.getBytes(UTF_8))) {
-                    credential = GoogleCredentials.fromStream(inputStream).createScoped(CLOUD_PLATFORM_SCOPE);
+                    credential = GoogleCredential.fromStream(inputStream).createScoped(DEFAULT_SCOPES);
                 }
             }
             else {
                 credential = jsonGoogleCredential.orElseThrow(() -> new IllegalStateException("GCS credentials not configured"));
             }
             return new Storage.Builder(httpTransport, JacksonFactory.getDefaultInstance(), new RetryHttpInitializer(credential, RetryHttpInitializerOptions.builder()
-                        .setReadTimeout(gcsOptions.getHttpRequestReadTimeout())
+                        .setReadTimeout(Duration.ofMillis(gcsOptions.getHttpRequestReadTimeout()))
                         .setMaxRequestRetries(gcsOptions.getMaxHttpRequestRetries())
-                        .build()))
+                    .build()))
                     .setApplicationName(APPLICATION_NAME)
                     .build();
         }

diff --git a/lib/trino-hdfs/src/main/java/io/trino/hdfs/gcs/GoogleGcsConfigurationInitializer.java b/lib/trino-hdfs/src/main/java/io/trino/hdfs/gcs/GoogleGcsConfigurationInitializer.java
@@ -29,11 +29,9 @@
 import java.util.Optional;
 
 import static com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystemConfiguration.GCS_CONFIG_PREFIX;
-import static com.google.cloud.hadoop.fs.gcs.HadoopCredentialsConfiguration.SERVICE_ACCOUNT_JSON_KEYFILE_SUFFIX;
-import static com.google.cloud.hadoop.repackaged.gcs.com.google.cloud.hadoop.util.HadoopCredentialsConfiguration.ACCESS_TOKEN_PROVIDER_SUFFIX;
-import static com.google.cloud.hadoop.repackaged.gcs.com.google.cloud.hadoop.util.HadoopCredentialsConfiguration.AUTHENTICATION_TYPE_SUFFIX;
-import static com.google.cloud.hadoop.repackaged.gcs.com.google.cloud.hadoop.util.HadoopCredentialsConfiguration.AuthenticationType.ACCESS_TOKEN_PROVIDER;
-import static com.google.cloud.hadoop.repackaged.gcs.com.google.cloud.hadoop.util.HadoopCredentialsConfiguration.AuthenticationType.SERVICE_ACCOUNT_JSON_KEYFILE;
+import static com.google.cloud.hadoop.repackaged.gcs.com.google.cloud.hadoop.util.HadoopCredentialConfiguration.ACCESS_TOKEN_PROVIDER_IMPL_SUFFIX;
+import static com.google.cloud.hadoop.repackaged.gcs.com.google.cloud.hadoop.util.HadoopCredentialConfiguration.ENABLE_SERVICE_ACCOUNTS_SUFFIX;
+import static com.google.cloud.hadoop.repackaged.gcs.com.google.cloud.hadoop.util.HadoopCredentialConfiguration.SERVICE_ACCOUNT_JSON_KEYFILE_SUFFIX;
 import static java.nio.file.attribute.PosixFilePermission.OWNER_READ;
 import static java.nio.file.attribute.PosixFilePermission.OWNER_WRITE;
 
@@ -74,12 +72,12 @@ public void initializeConfiguration(Configuration config)
 
         if (useGcsAccessToken) {
             // use oauth token to authenticate with Google Cloud Storage
-            config.setEnum(GCS_CONFIG_PREFIX + AUTHENTICATION_TYPE_SUFFIX.getKey(), ACCESS_TOKEN_PROVIDER);
-            config.setClass(GCS_CONFIG_PREFIX + ACCESS_TOKEN_PROVIDER_SUFFIX.getKey(), GcsAccessTokenProvider.class, AccessTokenProvider.class);
+            config.setBoolean(GCS_CONFIG_PREFIX + ENABLE_SERVICE_ACCOUNTS_SUFFIX.getKey(), false);
+            config.setClass(GCS_CONFIG_PREFIX + ACCESS_TOKEN_PROVIDER_IMPL_SUFFIX.getKey(), GcsAccessTokenProvider.class, AccessTokenProvider.class);
         }
         else if (jsonKeyFilePath != null) {
             // use service account key file
-            config.setEnum(GCS_CONFIG_PREFIX + AUTHENTICATION_TYPE_SUFFIX.getKey(), SERVICE_ACCOUNT_JSON_KEYFILE);
+            config.setBoolean(GCS_CONFIG_PREFIX + ENABLE_SERVICE_ACCOUNTS_SUFFIX.getKey(), true);
             config.set(GCS_CONFIG_PREFIX + SERVICE_ACCOUNT_JSON_KEYFILE_SUFFIX.getKey(), jsonKeyFilePath);
         }
     }

diff --git a/pom.xml b/pom.xml
@@ -522,7 +522,7 @@
             <dependency>
                 <groupId>com.google.cloud.bigdataoss</groupId>
                 <artifactId>gcs-connector</artifactId>
-                <version>3.0.0</version>
+                <version>hadoop3-2.2.19</version>
                 <classifier>shaded</classifier>
                 <exclusions>
                     <exclusion>