upgrade java-storage (#1001)

GoogleCloudDataproc · May 20, 2023 · 33dc155 · 33dc155
1 parent ed9a51e
commit 33dc155
Show file tree

Hide file tree

Showing 8 changed files with 23 additions and 4 deletions.
diff --git a/cloudbuild/cloudbuild.yaml b/cloudbuild/cloudbuild.yaml
@@ -44,6 +44,7 @@ steps:
       - 'VCS_COMMIT_ID=$COMMIT_SHA'
       - 'VCS_TAG=$TAG_NAME'
       - 'CI_BUILD_ID=$BUILD_ID'
+      - 'GCS_TEST_DIRECT_PATH_PREFERRED=false'
 
 # Tests take on average 25 minutes to run
 timeout: 2400s

diff --git a/gcs/CHANGES.md b/gcs/CHANGES.md
@@ -120,6 +120,8 @@
 
 1.  Upgrade Hadoop to 3.3.5.
 
+1.  Upgrade java-storage to 2.22.2
+
 ### 2.2.2 - 2021-06-25
 
 1.  Support footer prefetch in gRPC read channel.

diff --git a/...src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystemIntegrationHelper.java b/...src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystemIntegrationHelper.java
@@ -54,6 +54,8 @@ public static Configuration getTestConfig() {
           "fs.gs.auth.service.account.json.keyfile", testConf.getServiceAccountJsonKeyFile());
     }
 
+    config.setBoolean("fs.gs.grpc.directpath.enable", testConf.isDirectPathPreferred());
+
     return config;
   }
 

diff --git a/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/testing/TestConfiguration.java b/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/testing/TestConfiguration.java
@@ -21,6 +21,8 @@ public abstract class TestConfiguration {
   public static final String GCS_TEST_PROJECT_ID = "GCS_TEST_PROJECT_ID";
   public static final String GCS_TEST_JSON_KEYFILE = "GCS_TEST_JSON_KEYFILE";
 
+  public static final String GCS_TEST_DIRECT_PATH_PREFERRED = "GCS_TEST_DIRECT_PATH_PREFERRED";
+
   /** Environment-based test configuration. */
   public static class EnvironmentBasedTestConfiguration extends TestConfiguration {
     @Override
@@ -32,6 +34,16 @@ public String getProjectId() {
     public String getServiceAccountJsonKeyFile() {
       return System.getenv(GCS_TEST_JSON_KEYFILE);
     }
+
+    @Override
+    public boolean isDirectPathPreferred() {
+      String envVar = System.getenv(GCS_TEST_DIRECT_PATH_PREFERRED);
+      // if env variable is not configured default behaviour is to attempt directPath
+      if (envVar == null) {
+        return true;
+      }
+      return Boolean.parseBoolean(envVar);
+    }
   }
 
   public static TestConfiguration getInstance() {
@@ -45,4 +57,6 @@ private static class LazyHolder {
   public abstract String getProjectId();
 
   public abstract String getServiceAccountJsonKeyFile();
+
+  public abstract boolean isDirectPathPreferred();
 }
diff --git a/.../test/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageFileSystemIntegrationTest.java b/.../test/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageFileSystemIntegrationTest.java
@@ -107,6 +107,7 @@ public void before() throws Exception {
               GoogleCloudStorageOptions.builder()
                   .setAppName(GoogleCloudStorageTestHelper.APP_NAME)
                   .setProjectId(projectId)
+                  .setDirectPathPreferred(TestConfiguration.getInstance().isDirectPathPreferred())
                   .setWriteChannelOptions(
                       AsyncWriteChannelOptions.builder()
                           .setUploadChunkSize(64 * 1024 * 1024)

diff --git a/...src/test/java/com/google/cloud/hadoop/gcsio/integration/GoogleCloudStorageTestHelper.java b/...src/test/java/com/google/cloud/hadoop/gcsio/integration/GoogleCloudStorageTestHelper.java
@@ -110,6 +110,7 @@ public static Credentials getCredentials() throws IOException {
   public static GoogleCloudStorageOptions.Builder getStandardOptionBuilder() {
     return GoogleCloudStorageOptions.builder()
         .setAppName(GoogleCloudStorageTestHelper.APP_NAME)
+        .setDirectPathPreferred(TestConfiguration.getInstance().isDirectPathPreferred())
         .setProjectId(checkNotNull(TestConfiguration.getInstance().getProjectId()));
   }
 

diff --git a/pom.xml b/pom.xml
@@ -89,10 +89,10 @@
     <google.auth.version>1.14.0</google.auth.version>
     <google.auto-value.version>1.10.1</google.auto-value.version>
     <google.cloud-core.verion>2.9.0</google.cloud-core.verion>
-    <google.cloud-storage.bom.version>2.22.1</google.cloud-storage.bom.version>
+    <google.cloud-storage.bom.version>2.22.2</google.cloud-storage.bom.version>
     <google.error-prone.version>2.16</google.error-prone.version>
     <google.flogger.version>0.7.4</google.flogger.version>
-    <google.gax.version>2.20.1</google.gax.version>
+    <google.gax.version>2.27.0</google.gax.version>
     <google.gson.version>2.10</google.gson.version>
     <google.guava.version>31.1-jre</google.guava.version>
     <google.http-client.version>1.42.3</google.http-client.version>

diff --git a/tools/run_integration_tests.sh b/tools/run_integration_tests.sh
@@ -71,7 +71,5 @@ fi
 export GCS_TEST_JSON_KEYFILE
 export HDFS_ROOT=file:///tmp
 export RUN_INTEGRATION_TESTS=true
-# Env variable to enable direct path over gRPC
-export GOOGLE_CLOUD_ENABLE_DIRECT_PATH_XDS=true
 
 ./mvnw -B -e -T1C -Pintegration-test clean verify "${@:3}"