diff --git a/extended/src/test/java/apoc/load/LoadS3Test.java b/extended/src/test/java/apoc/load/LoadS3Test.java index 0462dd2493..bb6e910b24 100644 --- a/extended/src/test/java/apoc/load/LoadS3Test.java +++ b/extended/src/test/java/apoc/load/LoadS3Test.java @@ -2,13 +2,10 @@ import apoc.util.TestUtil; import apoc.util.Util; -import apoc.util.s3.S3Container; +import apoc.util.s3.S3BaseTest; import apoc.xml.XmlTestUtils; -import org.junit.After; -import org.junit.AfterClass; import org.junit.Assert; import org.junit.Before; -import org.junit.BeforeClass; import org.junit.Rule; import org.junit.Test; import org.junit.jupiter.api.AfterAll; @@ -27,37 +24,16 @@ import static java.util.Arrays.asList; import static org.junit.Assert.assertEquals; -public class LoadS3Test { +public class LoadS3Test extends S3BaseTest { @Rule public DbmsRule db = new ImpermanentDbmsRule(); - private S3Container minio; - - @BeforeClass - public static void init() { - // In test environment we skip the MD5 validation that can cause issues - System.setProperty("com.amazonaws.services.s3.disableGetObjectMD5Validation", "true"); - System.setProperty("com.amazonaws.sdk.disableCertChecking", "true"); - } - - @AfterClass - public static void destroy() { - System.clearProperty("com.amazonaws.services.s3.disableGetObjectMD5Validation"); - System.clearProperty("com.amazonaws.sdk.disableCertChecking"); - } - @Before public void setUp() throws Exception { TestUtil.registerProcedure(db, LoadCsv.class, LoadJson.class, Xml.class); apocConfig().setProperty(APOC_IMPORT_FILE_ENABLED, true); apocConfig().setProperty(APOC_IMPORT_FILE_USE_NEO4J_CONFIG, false); - minio = new S3Container(); - } - - @After - public void tearDown() throws Exception { - minio.close(); } @AfterAll @@ -67,7 +43,7 @@ public void tearDownAll() { @Test public void testLoadCsvS3() throws Exception { - String url = minio.putFile("src/test/resources/test.csv"); + String url = s3Container.putFile("src/test/resources/test.csv"); testResult(db, "CALL apoc.load.csv($url,{failOnError:false})", map("url", url), (r) -> { assertRow(r, "Selma", "8", 0L); assertRow(r, "Rana", "11", 1L); @@ -77,7 +53,7 @@ public void testLoadCsvS3() throws Exception { } @Test public void testLoadJsonS3() throws Exception { - String url = minio.putFile("src/test/resources/map.json"); + String url = s3Container.putFile("src/test/resources/map.json"); testCall(db, "CALL apoc.load.json($url,'')",map("url", url), (row) -> { @@ -86,7 +62,7 @@ public void testLoadCsvS3() throws Exception { } @Test public void testLoadXmlS3() throws Exception { - String url = minio.putFile("src/test/resources/xml/books.xml"); + String url = s3Container.putFile("src/test/resources/xml/books.xml"); testCall(db, "CALL apoc.load.xml($url,'/catalog/book[title=\"Maeve Ascendant\"]/.',{failOnError:false}) yield value as result", Util.map("url", url), (r) -> { Object value = Iterables.single(r.values()); diff --git a/extended/src/test/java/apoc/util/s3/S3Container.java b/extended/src/test/java/apoc/util/s3/S3Container.java deleted file mode 100644 index 974d7e1b9f..0000000000 --- a/extended/src/test/java/apoc/util/s3/S3Container.java +++ /dev/null @@ -1,120 +0,0 @@ -package apoc.util.s3; - -import apoc.util.JsonUtil; -import apoc.util.TestUtil; -import com.amazonaws.auth.AWSCredentialsProvider; -import com.amazonaws.client.builder.AwsClientBuilder; -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.AmazonS3ClientBuilder; -import com.amazonaws.services.s3.model.S3Object; -import com.amazonaws.services.s3.model.S3ObjectInputStream; -import org.apache.commons.io.FileUtils; -import org.testcontainers.containers.localstack.LocalStackContainer; - -import java.io.File; -import java.io.IOException; -import java.net.URL; -import java.nio.file.Paths; - -import static org.junit.Assert.assertEquals; -import static org.testcontainers.containers.localstack.LocalStackContainer.Service.S3; - -public class S3Container implements AutoCloseable { - private final static String S3_BUCKET_NAME = "test-bucket"; - private LocalStackContainer localstack; - private final AmazonS3 s3; - - - public S3Container() { - localstack = new LocalStackContainer("0.8.10") - .withStartupAttempts(3) - .withServices(S3); - localstack.start(); - s3 = AmazonS3ClientBuilder - .standard() - .withEndpointConfiguration(getEndpointConfiguration()) - .withCredentials(getCredentialsProvider()) - .build(); - s3.createBucket(getBucket()); - } - - public void close() { - localstack.close(); - s3.shutdown(); - } - - public AwsClientBuilder.EndpointConfiguration getEndpointConfiguration() { - return localstack.getEndpointConfiguration(S3); - } - - public AWSCredentialsProvider getCredentialsProvider() { - return localstack.getDefaultCredentialsProvider(); - } - - public String getBucket() { - return S3_BUCKET_NAME; - } - - public String getUrl(String key) { - return String.format("s3://%s.%s/%s/%s?accessKey=%s&secretKey=%s", - localstack.getEndpointConfiguration(S3).getSigningRegion(), - localstack.getEndpointConfiguration(S3).getServiceEndpoint() - .replace("http://", ""), - S3_BUCKET_NAME, - key, - localstack.getDefaultCredentialsProvider().getCredentials().getAWSAccessKeyId(), - localstack.getDefaultCredentialsProvider().getCredentials().getAWSSecretKey()); - } - - public String putFile(String fileName) { - final File file = new File(fileName); - s3.putObject(getBucket(), file.getName(), file); - return getUrl(file.getName()); - } - - public void verifyUpload(File directory, String fileName, String expected) throws IOException { - String s3Url = getUrl(fileName); - S3Container.readFile(s3Url, Paths.get(directory.toString(), fileName).toString()); - assertEquals(expected, readFileToString(directory.getAbsolutePath(), fileName)); - } - - public static void verifyUpload(File directoryExpected, File directory, String s3Url, String fileName) throws IOException { - readFile(s3Url, Paths.get(directory.toString(), fileName).toString()); - assertFileEquals(directoryExpected, fileName); - } - - public static void assertFileEquals(File directoryExpected, String fileName) { - String actualText = TestUtil.readFileToString(new File(directoryExpected, fileName)); - assertStreamEquals(directoryExpected, fileName, actualText); - } - - public static void assertStreamEquals(File directoryExpected, String fileName, String actualText) { - String expectedText = TestUtil.readFileToString(new File(directoryExpected, fileName)); - String[] actualArray = actualText.split("\n"); - String[] expectArray = expectedText.split("\n"); - assertEquals(expectArray.length, actualArray.length); - for (int i = 0; i < actualArray.length; i++) { - assertEquals(JsonUtil.parse(expectArray[i],null, Object.class), JsonUtil.parse(actualArray[i],null, Object.class)); - } - } - - public static void readFile(String s3Url, String pathname) throws IOException { - S3Params s3Params = S3ParamsExtractor.extract(new URL(s3Url)); - S3Aws s3Aws = new S3Aws(s3Params, s3Params.getRegion()); - AmazonS3 s3Client = s3Aws.getClient(); - - S3Object s3object = s3Client.getObject(s3Params.getBucket(), s3Params.getKey()); - S3ObjectInputStream inputStream = s3object.getObjectContent(); - FileUtils.copyInputStreamToFile(inputStream, new File(pathname)); - } - - private static String readFileToString(String directory, String fileName) { - return TestUtil.readFileToString(new File(directory, fileName)); - } - - public boolean isRunning() { - return localstack != null ? localstack.isRunning() : false; - } - - -}