upgrade google-java-storage (#997) (#1000)

gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystemHTTPClientIntegrationTest.java

@@ -0,0 +1,32 @@
+/*
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.hadoop.fs.gcs;
+
+import com.google.cloud.hadoop.gcsio.GoogleCloudStorageFileSystemOptions.ClientType;
+import org.junit.Before;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+@RunWith(JUnit4.class)
+public class GoogleHadoopFileSystemHTTPClientIntegrationTest
+    extends GoogleHadoopFileSystemIntegrationTest {
+  @Before
+  public void before() throws Exception {
+    storageClientType = ClientType.HTTP_API_CLIENT;
+    super.before();
+  }
+}

gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystemIntegrationTest.java

@@ -91,12 +91,9 @@
 import org.junit.Before;
 import org.junit.Ignore;
 import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
 
 /** Integration tests for GoogleHadoopFileSystem class. */
-@RunWith(Parameterized.class)
-public class GoogleHadoopFileSystemIntegrationTest extends GoogleHadoopFileSystemTestBase {
+public abstract class GoogleHadoopFileSystemIntegrationTest extends GoogleHadoopFileSystemTestBase {
 
   private static final String PUBLIC_BUCKET = "gs://gcp-public-data-landsat";
 

gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystemJavaStorageClientIntegrationTest.java

@@ -0,0 +1,64 @@
+package com.google.cloud.hadoop.fs.gcs;
+/*
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import com.google.cloud.hadoop.gcsio.GoogleCloudStorageFileSystemOptions.ClientType;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+// TODO: Ignored test for gRPC-directpath
+// https://github.com/GoogleCloudDataproc/hadoop-connectors/issues/998
+@RunWith(JUnit4.class)
+public class GoogleHadoopFileSystemJavaStorageClientIntegrationTest
+    extends GoogleHadoopFileSystemIntegrationTest {
+
+  @Before
+  public void before() throws Exception {
+    storageClientType = ClientType.STORAGE_CLIENT;
+    super.before();
+  }
+
+  @Ignore
+  @Test
+  public void testImpersonationGroupNameIdentifierUsed() {}
+
+  @Ignore
+  @Test
+  public void testImpersonationServiceAccountAndUserAndGroupNameIdentifierUsed() {}
+
+  @Ignore
+  @Test
+  public void testImpersonationServiceAccountUsed() {}
+
+  @Ignore
+  @Test
+  public void testImpersonationUserAndGroupNameIdentifiersUsed() {}
+
+  @Ignore
+  @Test
+  public void testImpersonationUserNameIdentifierUsed() {}
+
+  @Ignore
+  @Test
+  public void unauthenticatedAccessToPublicBuckets_fsGsProperties() {}
+
+  @Ignore
+  @Test
+  public void unauthenticatedAccessToPublicBuckets_googleCloudProperties() {}
+}

gcsio/src/test/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageFileSystemHTTPClientTest.java

@@ -0,0 +1,33 @@
+/*
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.hadoop.gcsio;
+
+import com.google.cloud.hadoop.gcsio.GoogleCloudStorageFileSystemOptions.ClientType;
+import org.junit.Before;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+@RunWith(JUnit4.class)
+public class GoogleCloudStorageFileSystemHTTPClientTest
+    extends GoogleCloudStorageFileSystemTestBase {
+
+  @Before
+  public void before() throws Exception {
+    storageClientType = ClientType.HTTP_API_CLIENT;
+    super.before();
+  }
+}

gcsio/src/test/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageFileSystemIntegrationTest.java

@@ -432,7 +432,8 @@ public void testGetAndListFileInfo() throws Exception {
     // At o1.
     validateListFileInfo(testBucket, "o1", /* expectedToExist= */ true, "o1");
     if (getClass().equals(GoogleCloudStorageFileSystemIntegrationTest.class)
-        || getClass().equals(GoogleCloudStorageFileSystemTest.class)) {
+        || getClass().equals(GoogleCloudStorageFileSystemHTTPClientTest.class)
+        || getClass().equals(GoogleCloudStorageFileSystemJavaStorageClientTest.class)) {
       validateListFileInfo(testBucket, "o1/", /* expectedToExist= */ false);
     } else {
       validateListFileInfo(testBucket, "o1/", /* expectedToExist= */ true, "o1");
@@ -447,7 +448,8 @@ public void testGetAndListFileInfo() throws Exception {
     // At d1/o12.
     validateListFileInfo(testBucket, "d1/o12", /* expectedToExist= */ true, "d1/o12");
     if (getClass().equals(GoogleCloudStorageFileSystemIntegrationTest.class)
-        || getClass().equals(GoogleCloudStorageFileSystemTest.class)) {
+        || getClass().equals(GoogleCloudStorageFileSystemHTTPClientTest.class)
+        || getClass().equals(GoogleCloudStorageFileSystemJavaStorageClientTest.class)) {
       validateListFileInfo(testBucket, "d1/o12/", /* expectedToExist= */ false);
     } else {
       validateListFileInfo(testBucket, "d1/o12/", /* expectedToExist= */ true, "d1/o12");

gcsio/src/test/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageFileSystemJavaStorageClientTest.java

@@ -0,0 +1,46 @@
+/*
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.hadoop.gcsio;
+
+import com.google.cloud.hadoop.gcsio.GoogleCloudStorageFileSystemOptions.ClientType;
+import java.io.IOException;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+@RunWith(JUnit4.class)
+public class GoogleCloudStorageFileSystemJavaStorageClientTest
+    extends GoogleCloudStorageFileSystemTestBase {
+
+  @Before
+  public void before() throws Exception {
+    storageClientType = ClientType.STORAGE_CLIENT;
+    super.before();
+  }
+
+  @Override
+  @Ignore("DirectPath is not supported with null credentials")
+  @Test
+  public void testConstructor() throws IOException {}
+
+  @Override
+  @Ignore("DirectPath is not supported with null credentials")
+  @Test
+  public void testClientType() {}
+}

gcsio/src/test/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageFileSystemTest.java → gcsio/src/test/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageFileSystemTestBase.java

@@ -36,16 +36,14 @@
 import java.util.logging.Logger;
 import org.junit.Before;
 import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
 
 /**
  * The unittest version of {@code GoogleCloudStorageFileSystemIntegrationTest}; the external
  * GoogleCloudStorage dependency is replaced by an in-memory version which mimics the same
  * bucket/object semantics.
  */
-@RunWith(Parameterized.class)
-public class GoogleCloudStorageFileSystemTest extends GoogleCloudStorageFileSystemIntegrationTest {
+public abstract class GoogleCloudStorageFileSystemTestBase
+    extends GoogleCloudStorageFileSystemIntegrationTest {
 
   @Before
   public void before() throws Exception {

pom.xml

@@ -89,7 +89,7 @@
     <google.auth.version>1.14.0</google.auth.version>
     <google.auto-value.version>1.10.1</google.auto-value.version>
     <google.cloud-core.verion>2.9.0</google.cloud-core.verion>
-    <google.cloud-storage.bom.version>2.17.2</google.cloud-storage.bom.version>
+    <google.cloud-storage.bom.version>2.22.1</google.cloud-storage.bom.version>
     <google.error-prone.version>2.16</google.error-prone.version>
     <google.flogger.version>0.7.4</google.flogger.version>
     <google.gax.version>2.20.1</google.gax.version>
@@ -99,7 +99,7 @@
     <google.java-format.version>1.15.0</google.java-format.version>
     <google.oauth-client.version>1.34.1</google.oauth-client.version>
     <google.protobuf.version>3.21.12</google.protobuf.version>
-    <grpc.version>1.52.1</grpc.version>
+    <grpc.version>1.54.0</grpc.version>
     <hadoop.version>3.3.5</hadoop.version>
     <opencensus.version>0.31.1</opencensus.version>
 

tools/run_integration_tests.sh

@@ -71,5 +71,7 @@ fi
 export GCS_TEST_JSON_KEYFILE
 export HDFS_ROOT=file:///tmp
 export RUN_INTEGRATION_TESTS=true
+# Env variable to enable direct path over gRPC
+export GOOGLE_CLOUD_ENABLE_DIRECT_PATH_XDS=true
 
 ./mvnw -B -e -T1C -Pintegration-test clean verify "${@:3}"