Skip to content

Commit

Permalink
Issue-60: Set up testcontainers
Browse files Browse the repository at this point in the history 
- Use a fixed docker image version of minio.
- Auto create minio bucket.
- Minor fixes and improvements.
  • Loading branch information
@steve-todorov
steve-todorov committed Jan 31, 2021
1 parent ed97fe0 commit 035fc7a
Show file tree
Hide file tree
Showing 4 changed files with 72 additions and 31 deletions.
18 changes: 13 additions & 5 deletions src/test/java/org/carlspring/cloud/storage/s3fs/util/BaseIntegrationTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,14 +10,22 @@ public abstract class BaseIntegrationTest
{
if (isMinioEnv())
{
final String accessKey = (String) EnvironmentBuilder.getRealEnv().get(ACCESS_KEY);
final String secretKey = (String) EnvironmentBuilder.getRealEnv().get(SECRET_KEY);
final MinioContainer minioContainer = new MinioContainer(accessKey, secretKey);
minioContainer.start();
try
{
final String accessKey = (String) EnvironmentBuilder.getRealEnv().get(ACCESS_KEY);
final String secretKey = (String) EnvironmentBuilder.getRealEnv().get(SECRET_KEY);
final String bucketName = (String) EnvironmentBuilder.getRealEnv().get(EnvironmentBuilder.BUCKET_NAME_KEY);
final MinioContainer minioContainer = new MinioContainer(accessKey, secretKey, bucketName);
minioContainer.start();
}
catch (Exception e)
{
e.printStackTrace();
}
}
}

private static boolean isMinioEnv()
public static boolean isMinioEnv()
{
String integrationTestType = System.getProperty("running.it");
return integrationTestType != null && integrationTestType.equalsIgnoreCase("minio");
Expand Down
69 changes: 46 additions & 23 deletions src/test/java/org/carlspring/cloud/storage/s3fs/util/EnvironmentBuilder.java
View file
Original file line number Diff line number Diff line change
@@ -1,21 +1,23 @@
package org.carlspring.cloud.storage.s3fs.util;

import javax.annotation.Nonnull;
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.Map;
import java.util.Properties;

import com.google.common.collect.ImmutableMap;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.client.utils.URIBuilder;

import javax.annotation.Nonnull;

import static org.carlspring.cloud.storage.s3fs.S3Factory.ACCESS_KEY;
import static org.carlspring.cloud.storage.s3fs.S3Factory.PROTOCOL;
import static org.carlspring.cloud.storage.s3fs.S3Factory.REGION;
import static org.carlspring.cloud.storage.s3fs.S3Factory.SECRET_KEY;
import static org.carlspring.cloud.storage.s3fs.util.S3EndpointConstant.MINIO_GLOBAL_URI_IT;
import static org.carlspring.cloud.storage.s3fs.util.S3EndpointConstant.S3_REGION_URI_IT;

/**
Expand Down Expand Up @@ -45,36 +47,50 @@ public static Map<String, Object> getRealEnv()
if (accessKey != null && secretKey != null && region != null && protocol != null && bucket != null)
{
env = ImmutableMap.<String, Object>builder().put(ACCESS_KEY, accessKey)
.put(SECRET_KEY, secretKey)
.put(REGION, region)
.put(PROTOCOL, protocol)
.put(BUCKET_NAME_KEY, bucket)
.build();
.put(SECRET_KEY, secretKey)
.put(REGION, region)
.put(PROTOCOL, protocol)
.put(BUCKET_NAME_KEY, bucket)
.build();
}
else
{
final Properties props = new Properties();

try
{
props.load(EnvironmentBuilder.class.getResourceAsStream("/amazon-test.properties"));
props.load(getPropertiesResource());
}
catch (IOException e)
{
throw new RuntimeException("not found amazon-test.properties in the classpath", e);
throw new RuntimeException("Unable to load properties file from classpath nor to find environment variables!", e);
}

env = ImmutableMap.<String, Object>builder().put(ACCESS_KEY, props.getProperty(ACCESS_KEY))
.put(SECRET_KEY, props.getProperty(SECRET_KEY))
.put(REGION, props.getProperty(REGION))
.put(PROTOCOL, props.getProperty(PROTOCOL))
.put(BUCKET_NAME_KEY, props.getProperty(BUCKET_NAME_KEY))
.build();
.put(SECRET_KEY, props.getProperty(SECRET_KEY))
.put(REGION, props.getProperty(REGION))
.put(PROTOCOL, props.getProperty(PROTOCOL))
.put(BUCKET_NAME_KEY, props.getProperty(BUCKET_NAME_KEY))
.build();
}

return env;
}

private static InputStream getPropertiesResource()
throws IOException
{
URL resourceUrl = EnvironmentBuilder.class.getResource("/amazon-test.properties");
if(resourceUrl != null)
{
return EnvironmentBuilder.class.getResourceAsStream("/amazon-test.properties");
}
else
{
throw new IOException("File [classpath:/amazon-test.properties] not found!");
}
}

/**
* Attempt to retrieve OS Environment Variable
* @return
Expand All @@ -99,12 +115,6 @@ private static String getS3EnvName(@Nonnull String key)
sanitized = StringUtils.removeStartIgnoreCase(sanitized, "S3FS_");
sanitized = sanitized.replaceAll("\\.", "_").toUpperCase();

String integrationSuite = System.getProperty("running.it");
if(integrationSuite != null && integrationSuite.equalsIgnoreCase("minio"))
{
sanitized = "MINIO_" + sanitized;
}

sanitized = "S3FS_" + sanitized;

return sanitized;
Expand Down Expand Up @@ -132,7 +142,7 @@ public static String getBucket()

try
{
props.load(EnvironmentBuilder.class.getResourceAsStream("/amazon-test.properties"));
props.load(getPropertiesResource());

bucketName = props.getProperty(BUCKET_NAME_KEY);
if (bucketName != null && !bucketName.endsWith("/"))
Expand All @@ -144,7 +154,7 @@ public static String getBucket()
}
catch (IOException e)
{
throw new RuntimeException("needed /amazon-test.properties in the classpath");
throw new RuntimeException("Unable to load properties file from classpath nor to find environment variables!", e);
}
}

Expand All @@ -163,7 +173,20 @@ public static URI getS3URI(URI s3GlobalUri)
final String accessKey = (String) env.get(ACCESS_KEY);
final String secretKey = (String) env.get(SECRET_KEY);
final String region = (String) env.get(REGION);
final URI s3Uri = region != null ? URI.create(String.format(S3_REGION_URI_IT, region)) : s3GlobalUri;

URI s3Uri;

// When we're running -Pit-s3 - proceed as usual.
if(!BaseIntegrationTest.isMinioEnv())
{
s3Uri = region != null ? URI.create(String.format(S3_REGION_URI_IT, region)) : s3GlobalUri;
}
// When we're running -Pit-minio - overwrite any region configuration to localhost:9000 where minio is running.
else
{
s3Uri = MINIO_GLOBAL_URI_IT;
}

return new URIBuilder(s3Uri).setUserInfo(accessKey, secretKey)
.build();
}
Expand Down
14 changes: 11 additions & 3 deletions src/test/java/org/carlspring/cloud/storage/s3fs/util/MinioContainer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,21 +12,29 @@ public class MinioContainer

private static final int DEFAULT_PORT = 9000;
private static final String DEFAULT_IMAGE = "minio/minio";
private static final String DEFAULT_TAG = "edge";
private static final String DEFAULT_TAG = "RELEASE.2021-01-30T00-20-58Z";
private static final String MINIO_ACCESS_KEY = "MINIO_ACCESS_KEY";
private static final String MINIO_SECRET_KEY = "MINIO_SECRET_KEY";
private static final String DEFAULT_STORAGE_DIRECTORY = "/data";
private static final String HEALTH_ENDPOINT = "/minio/health/ready";

public MinioContainer(String accessKey,
String secretKey)
String secretKey,
String bucketName)
{
super(DEFAULT_IMAGE + ":" + DEFAULT_TAG);
withNetworkAliases("minio-" + Base58.randomString(6));
addExposedPort(DEFAULT_PORT);
withEnv(MINIO_ACCESS_KEY, accessKey);
withEnv(MINIO_SECRET_KEY, secretKey);
withCommand("server", DEFAULT_STORAGE_DIRECTORY);

// Auto create bucket on start up because minio does not do this for you.
// https://github.com/minio/minio/issues/4769#issuecomment-331033735
withCreateContainerCmdModifier(cmd -> {
cmd.withEntrypoint("/bin/sh");
cmd.withCmd("-c", "mkdir -p /data" + bucketName + " && minio server " + DEFAULT_STORAGE_DIRECTORY);
});

setWaitStrategy(new HttpWaitStrategy().forPort(DEFAULT_PORT)
.forPath(HEALTH_ENDPOINT)
.withStartupTimeout(Duration.ofMinutes(2)));
Expand Down
2 changes: 2 additions & 0 deletions src/test/java/org/carlspring/cloud/storage/s3fs/util/S3EndpointConstant.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@ public class S3EndpointConstant

public static final URI S3_GLOBAL_URI_IT = URI.create("s3://s3.amazonaws.com/");

public static final URI MINIO_GLOBAL_URI_IT = URI.create("s3://localhost:9000");

public static final String S3_REGION_URI_IT = "s3://s3.%s.amazonaws.com/";

}

0 comments on commit 035fc7a

Please sign in to comment.