Skip to content

Commit

Permalink
fix: add runtime hints for storage (#2001)
Browse files Browse the repository at this point in the history 
* fix: add runtime hints for sample and add sample test to ci; refactor tests to use unique resources

---------

Co-authored-by: mpeddada1 <[email protected]>
Co-authored-by: Mridula <[email protected]>
  • Loading branch information
@zhumin8 @mpeddada1
3 people authored Aug 4, 2023
1 parent 2ba4a75 commit 793c798
Show file tree
Hide file tree
Showing 11 changed files with 232 additions and 54 deletions.
2 changes: 2 additions & 0 deletions pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -353,6 +353,7 @@
<profile>
<id>spring-native</id>
<modules>
<module>spring-cloud-gcp-storage</module>
<module>spring-cloud-gcp-vision</module>
</modules>

Expand All @@ -379,6 +380,7 @@
<classesDirectory>${project.build.outputDirectory}</classesDirectory>
<systemPropertyVariables>
<!--integration tests are not invoked unless the relevant system property is set to true here. -->
<it.storage>true</it.storage>
<it.vision>true</it.vision>
</systemPropertyVariables>
</configuration>
Expand Down
8 changes: 8 additions & 0 deletions spring-cloud-gcp-samples/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,8 @@
<id>native-sample-config</id>
<modules>
<module>spring-cloud-gcp-logging-sample</module>
<module>spring-cloud-gcp-storage-resource-sample</module>
<module>spring-cloud-gcp-integration-storage-sample</module>
<module>spring-cloud-gcp-trace-sample</module>
<module>spring-cloud-gcp-vision-api-sample</module>
</modules>
Expand Down Expand Up @@ -135,8 +137,14 @@
</includes>
<systemPropertyVariables>
<it.logging>true</it.logging>
<it.storage>true</it.storage>
<it.vision>true</it.vision>
<it.trace>true</it.trace>
<it.vision>true</it.vision>
<gcs-resource-test-bucket>gcp-storage-resource-bucket-sample</gcs-resource-test-bucket>
<gcs-read-bucket>gcp-storage-bucket-sample-input</gcs-read-bucket>
<gcs-write-bucket>gcp-storage-bucket-sample-output</gcs-write-bucket>
<gcs-local-directory>/tmp/gcp_integration_tests/integration_storage_sample</gcs-local-directory>
</systemPropertyVariables>
</configuration>
</plugin>
Expand Down
17 changes: 10 additions & 7 deletions ...integration-storage-sample/src/main/java/com/example/GcsSpringIntegrationApplication.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,21 +44,23 @@
@SpringBootApplication
public class GcsSpringIntegrationApplication {

private static final Log LOGGER = LogFactory.getLog(GcsSpringIntegrationApplication.class);

@Value("${gcs-read-bucket}")
private String gcsReadBucket;

@Value("${gcs-write-bucket}")
private String gcsWriteBucket;

@Value("${gcs-local-directory}")
private String localDirectory;

private static final Log LOGGER = LogFactory.getLog(GcsSpringIntegrationApplication.class);

public static void main(String[] args) {
SpringApplication.run(GcsSpringIntegrationApplication.class, args);
}

@Bean("localDirectoryName")
public String localDirectory(@Value("${gcs-local-directory}") String localDirectory) {
return localDirectory;
}

/**
* A file synchronizer that knows how to connect to the remote file system (GCS) and scan it for
* new files and then download the files.
Expand All @@ -83,10 +85,11 @@ public GcsInboundFileSynchronizer gcsInboundFileSynchronizer(Storage gcs) {
*/
@Bean
@InboundChannelAdapter(channel = "new-file-channel", poller = @Poller(fixedDelay = "5000"))
public MessageSource<File> synchronizerAdapter(GcsInboundFileSynchronizer synchronizer) {
public MessageSource<File> synchronizerAdapter(
GcsInboundFileSynchronizer synchronizer, String localDirectoryName) {
GcsInboundFileSynchronizingMessageSource syncAdapter =
new GcsInboundFileSynchronizingMessageSource(synchronizer);
syncAdapter.setLocalDirectory(Paths.get(this.localDirectory).toFile());
syncAdapter.setLocalDirectory(Paths.get(localDirectoryName).toFile());

return syncAdapter;
}
Expand Down
38 changes: 38 additions & 0 deletions ...ation-storage-sample/src/test/java/com/example/GcsSpringIntegrationTestConfiguration.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
/*
* Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.example;

import java.util.UUID;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.test.context.TestConfiguration;
import org.springframework.context.annotation.Bean;

@TestConfiguration
public class GcsSpringIntegrationTestConfiguration {

private String uniqueDirectory;

public GcsSpringIntegrationTestConfiguration(
@Value("${gcs-local-directory}") String localDirectory) {
uniqueDirectory = String.format("%s-%s", localDirectory, UUID.randomUUID());
}

@Bean("localDirectoryName")
public String uniqueDirectory() {
return uniqueDirectory;
}
}
78 changes: 48 additions & 30 deletions ...d-gcp-integration-storage-sample/src/test/java/com/example/GcsSpringIntegrationTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@

import static org.assertj.core.api.Assertions.assertThat;

import com.google.api.gax.paging.Page;
import com.google.cloud.storage.Blob;
import com.google.cloud.storage.BlobId;
import com.google.cloud.storage.BlobInfo;
Expand All @@ -30,15 +29,19 @@
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.awaitility.Awaitility;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestInstance;
import org.junit.jupiter.api.condition.EnabledIfSystemProperty;
import org.junit.jupiter.api.extension.ExtendWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.context.annotation.PropertySource;
Expand All @@ -54,44 +57,43 @@
@EnabledIfSystemProperty(named = "it.storage", matches = "true")
@ExtendWith(SpringExtension.class)
@PropertySource("classpath:application.properties")
@SpringBootTest(classes = {GcsSpringIntegrationApplication.class})
@SpringBootTest(
classes = {GcsSpringIntegrationApplication.class, GcsSpringIntegrationTestConfiguration.class},
properties = {"spring.main.allow-bean-definition-overriding=true"})
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
class GcsSpringIntegrationTests {

private static final String TEST_FILE_NAME = "test_file";
private static final String TEST_FILE = String.format("test_file_%s", UUID.randomUUID());
private static final Log LOGGER = LogFactory.getLog(GcsSpringIntegrationTests.class);

@Autowired private Storage storage;

@Autowired
@Qualifier("localDirectoryName")
private String uniqueDirectory;

@Value("${gcs-read-bucket}")
private String cloudInputBucket;

@Value("${gcs-write-bucket}")
private String cloudOutputBucket;

@Value("${gcs-local-directory}")
private String outputFolder;

@BeforeEach
void setupTestEnvironment() {
cleanupCloudStorage();
}

@AfterEach
@AfterAll
void teardownTestEnvironment() throws IOException {
cleanupCloudStorage();
cleanupLocalDirectory();
cleanupLocalDirectory(Paths.get(uniqueDirectory));
}

@Test
void testFilePropagatedToLocalDirectory() {
BlobId blobId = BlobId.of(this.cloudInputBucket, TEST_FILE_NAME);
BlobId blobId = BlobId.of(this.cloudInputBucket, TEST_FILE);
BlobInfo blobInfo = BlobInfo.newBuilder(blobId).setContentType("text/plain").build();
this.storage.create(blobInfo, "Hello World!".getBytes(StandardCharsets.UTF_8));

Awaitility.await()
.atMost(15, TimeUnit.SECONDS)
.atMost(30, TimeUnit.SECONDS)
.untilAsserted(
() -> {
Path outputFile = Paths.get(this.outputFolder + "/" + TEST_FILE_NAME);
Path outputFile = Paths.get(uniqueDirectory + "/" + TEST_FILE);
assertThat(Files.exists(outputFile)).isTrue();
assertThat(Files.isRegularFile(outputFile)).isTrue();

Expand All @@ -104,23 +106,39 @@ void testFilePropagatedToLocalDirectory() {
.iterateAll()
.forEach(b -> blobNamesInOutputBucket.add(b.getName()));

assertThat(blobNamesInOutputBucket).contains(TEST_FILE_NAME);
assertThat(blobNamesInOutputBucket).contains(TEST_FILE);
});
}

void cleanupCloudStorage() {
Page<Blob> blobs = this.storage.list(this.cloudInputBucket);
for (Blob blob : blobs.iterateAll()) {
blob.delete();
BlobId inputBucketBlobId = BlobId.of(cloudInputBucket, TEST_FILE);
Blob inputBucketBlob = storage.get(inputBucketBlobId);
if (inputBucketBlob != null) {
inputBucketBlob.delete();
}

BlobId outputBucketBlobId = BlobId.of(cloudOutputBucket, TEST_FILE);
Blob outputBucketBlob = storage.get(outputBucketBlobId);
if (outputBucketBlob != null) {
outputBucketBlob.delete();
}
}

void cleanupLocalDirectory() throws IOException {
Path localDirectory = Paths.get(this.outputFolder);
List<Path> files = Files.list(localDirectory).collect(Collectors.toList());
for (Path file : files) {
Files.delete(file);
void cleanupLocalDirectory(Path testDirectory) throws IOException {
if (Files.exists(testDirectory)) {
if (Files.isDirectory(testDirectory)) {
try (Stream<Path> files = Files.list(testDirectory)) {
files.forEach(
path -> {
try {
Files.delete(path);
} catch (IOException ioe) {
LOGGER.info("Error deleting test file.", ioe);
}
});
}
}
Files.delete(testDirectory);
}
Files.deleteIfExists(Paths.get(this.outputFolder));
}
}
1 change: 1 addition & 0 deletions ...es/spring-cloud-gcp-storage-resource-sample/src/main/java/com/example/GcsApplication.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;


/**
* An example Spring Boot application that reads and writes files stored in Google Cloud Storage
* (GCS) using the Spring Resource abstraction and the gs: protocol prefix.
Expand Down
43 changes: 37 additions & 6 deletions ...les/spring-cloud-gcp-storage-resource-sample/src/main/java/com/example/WebController.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,16 +16,20 @@

package com.example;

import com.google.cloud.spring.storage.GoogleStorageResource;
import com.google.cloud.storage.Storage;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.charset.Charset;
import java.util.Optional;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.core.io.Resource;
import org.springframework.core.io.WritableResource;
import org.springframework.util.StreamUtils;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;

/**
Expand All @@ -35,19 +39,46 @@
@RestController
public class WebController {

@Value("${gcs-resource-test-bucket}")
private String bucketName;

@Value("gs://${gcs-resource-test-bucket}/my-file.txt")
private Resource gcsFile;

@GetMapping("/")
public String readGcsFile() throws IOException {
return StreamUtils.copyToString(this.gcsFile.getInputStream(), Charset.defaultCharset()) + "\n";
private Storage storage;

private WebController(Storage storage) {
this.storage = storage;
}

@GetMapping(value = "/")
public String readGcsFile(@RequestParam("filename") Optional<String> filename)
throws IOException {
return StreamUtils.copyToString(
filename.isPresent()
? fetchResource(filename.get()).getInputStream()
: this.gcsFile.getInputStream(),
Charset.defaultCharset())
+ "\n";
}

@PostMapping("/")
public String writeGcs(@RequestBody String data) throws IOException {
try (OutputStream os = ((WritableResource) this.gcsFile).getOutputStream()) {
@PostMapping(value = "/")
public String writeGcs(
@RequestBody String data, @RequestParam("filename") Optional<String> filename)
throws IOException {
return updateResource(
filename.map(this::fetchResource).orElse((GoogleStorageResource) this.gcsFile), data);
}

private String updateResource(Resource resource, String data) throws IOException {
try (OutputStream os = ((WritableResource) resource).getOutputStream()) {
os.write(data.getBytes());
}
return "file was updated\n";
}

private GoogleStorageResource fetchResource(String filename) {
return new GoogleStorageResource(
this.storage, String.format("gs://%s/%s", this.bucketName, filename));
}
}
Loading

0 comments on commit 793c798

Please sign in to comment.