Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[4Bmcyc2U] Fix ignored S3 tests #241

Merged
merged 3 commits into from
Nov 29, 2022
Merged

[4Bmcyc2U] Fix ignored S3 tests #241

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
43a0a11
[4Bmcyc2U] Fix ignored S3 tests
vga91 Nov 17, 2022
5d648ab
fix tests
vga91 Nov 18, 2022
8f75d31
lightened ExportS3PerformanceTest
vga91 Nov 25, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion common/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,6 @@ dependencies {

// These dependencies affect the tests only, they will not be packaged in the resulting .jar
testImplementation project(':test-utils')
testImplementation group: 'com.amazonaws', name: 'aws-java-sdk-s3', version: '1.12.348'
testImplementation group: 'junit', name: 'junit', version: '4.13.2'
testImplementation group: 'com.github.stefanbirkner', name: 'system-rules', version: '1.19.0'
testImplementation group: 'org.hamcrest', name: 'hamcrest-library', version: '1.3'
Expand Down
32 changes: 0 additions & 32 deletions common/src/test/java/apoc/util/s3/S3TestUtil.java
View file
Open in desktop

This file was deleted.

61 changes: 15 additions & 46 deletions core/src/test/java/apoc/export/ExportS3PerformanceTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,70 +3,42 @@
import apoc.export.csv.ExportCSV;
import apoc.graph.Graphs;
import apoc.util.TestUtil;
import apoc.util.s3.S3Aws;
import apoc.util.s3.S3Params;
import apoc.util.s3.S3ParamsExtractor;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.model.ObjectListing;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import apoc.util.Util;
import apoc.util.s3.S3BaseTest;
import com.amazonaws.services.s3.model.S3Object;
import org.junit.BeforeClass;
import org.junit.ClassRule;
import org.junit.Test;
import org.junit.Ignore;
import org.neo4j.test.rule.DbmsRule;
import org.neo4j.test.rule.ImpermanentDbmsRule;

import java.io.IOException;
import java.net.URL;
import java.time.Duration;
import java.time.Instant;
import java.util.Optional;
import java.util.stream.IntStream;

import static apoc.ApocConfig.APOC_EXPORT_FILE_ENABLED;
import static apoc.ApocConfig.apocConfig;
import static apoc.util.MapUtil.map;
import static junit.framework.TestCase.assertTrue;
import static apoc.util.s3.S3TestUtil.getS3Object;
import static junit.framework.TestCase.assertEquals;

@Ignore("To use this test, you need to set the S3 bucket and region to a valid endpoint " +
"and have your access key and secret key setup in your environment.")
public class ExportS3PerformanceTest {
private static String S3_BUCKET_NAME = null;
private static int REPEAT_TEST = 3;

private static String getEnvVar(String envVarKey) throws Exception {
return Optional.ofNullable(System.getenv(envVarKey)).orElseThrow(
() -> new Exception(String.format("%s is not set in the environment", envVarKey))
);
}

private static String getS3Url(String key) {
return String.format("s3://:/%s/%s", S3_BUCKET_NAME, key);
}
public class ExportS3PerformanceTest extends S3BaseTest {
private final static int REPEAT_TEST = 3;

private void verifyFileUploaded(String s3Url, String fileName) throws IOException {
S3Params s3Params = S3ParamsExtractor.extract(new URL(s3Url));
S3Aws s3Aws = new S3Aws(s3Params, s3Params.getRegion());
AmazonS3 s3Client = s3Aws.getClient();

boolean fileUploaded = false;
ObjectListing objectListing = s3Client.listObjects(s3Params.getBucket());
for(S3ObjectSummary os : objectListing.getObjectSummaries()) {
if (os.getKey().equals(fileName)) {
fileUploaded = true;
break;
}
}
assertTrue(fileUploaded);
final S3Object s3Object = getS3Object(s3Url);
assertEquals(fileName, s3Object.getKey());
}

@ClassRule
public static DbmsRule db = new ImpermanentDbmsRule();

@BeforeClass
public static void setUp() throws Exception {
if (S3_BUCKET_NAME == null) {
S3_BUCKET_NAME = getEnvVar("S3_BUCKET_NAME");
}
baseBeforeClass();

TestUtil.registerProcedure(db, ExportCSV.class, Graphs.class);

apocConfig().setProperty(APOC_EXPORT_FILE_ENABLED, true);
Expand All @@ -75,17 +47,14 @@ public static void setUp() throws Exception {
@Test
public void testExportAllCsvS3() throws Exception {
System.out.println("Data creation started.");
// create large data (> 100 MB)
for (int i=0; i<555000; i++) {
String query = String.format("CREATE (f:User1:User {name:'foo%d',age:%d,male:true,kids:['a','b','c']})-[:KNOWS]->(b:User {name:'bar%d',age:%d}),(c:User {age:12})", i, i, i, i );
db.executeTransactionally(query);
}
final String query = Util.readResourceFile("movies.cypher");
IntStream.range(0, 5000).forEach(__-> db.executeTransactionally(query));
System.out.println("Data creation finished.");

System.out.println("Test started.");
for (int repeat=1; repeat<=REPEAT_TEST; repeat++) {
String fileName = String.format("performanceTest_%d.csv", repeat);
String s3Url = getS3Url(fileName);
String s3Url = s3Container.getUrl(fileName);

// Run the performance testing
final Instant start = Instant.now();
Expand Down
118 changes: 36 additions & 82 deletions core/src/test/java/apoc/export/csv/ExportCsvS3Test.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,31 +2,36 @@

import apoc.graph.Graphs;
import apoc.util.TestUtil;
import apoc.util.s3.S3TestUtil;
import apoc.util.s3.S3BaseTest;
import org.junit.BeforeClass;
import org.junit.ClassRule;
import org.junit.Test;
import org.junit.Ignore;
import org.neo4j.configuration.GraphDatabaseSettings;
import org.neo4j.test.rule.DbmsRule;
import org.neo4j.test.rule.ImpermanentDbmsRule;

import java.io.File;
import java.io.IOException;
import java.nio.file.Paths;
import java.util.Map;
import java.util.Optional;

import static apoc.ApocConfig.APOC_EXPORT_FILE_ENABLED;
import static apoc.ApocConfig.apocConfig;
import static apoc.util.MapUtil.map;
import static apoc.util.s3.S3TestUtil.assertStringFileEquals;
import static junit.framework.TestCase.assertTrue;
import static org.junit.Assert.assertEquals;

@Ignore("To use this test, you need to set the S3 bucket and region to a valid endpoint " +
"and have your access key and secret key setup in your environment.")
public class ExportCsvS3Test {
private static String S3_BUCKET_NAME = null;
public class ExportCsvS3Test extends S3BaseTest {

@ClassRule
public static DbmsRule db = new ImpermanentDbmsRule();

@BeforeClass
public static void setUp() throws Exception{
baseBeforeClass();

apocConfig().setProperty(APOC_EXPORT_FILE_ENABLED, true);
TestUtil.registerProcedure(db, ExportCSV.class, Graphs.class);
db.executeTransactionally("CREATE (f:User1:User {name:'foo',age:42,male:true,kids:['a','b','c']})-[:KNOWS]->(b:User {name:'bar',age:42}),(c:User {age:12})");
db.executeTransactionally("CREATE (f:Address1:Address {name:'Andrea', city: 'Milano', street:'Via Garibaldi, 7'})-[:NEXT_DELIVERY]->(a:Address {name: 'Bar Sport'}), (b:Address {street: 'via Benni'})");
}

private static final String EXPECTED_QUERY_NODES = String.format("\"u\"%n" +
"\"{\"\"id\"\":0,\"\"labels\"\":[\"\"User\"\",\"\"User1\"\"],\"\"properties\"\":{\"\"name\"\":\"\"foo\"\",\"\"age\"\":42,\"\"male\"\":true,\"\"kids\"\":[\"\"a\"\",\"\"b\"\",\"\"c\"\"]}}\"%n" +
Expand All @@ -40,18 +45,6 @@ public class ExportCsvS3Test {
"42,foo,true,[\"a\",\"b\",\"c\"],[\"User1\",\"User\"]%n" +
"42,bar,,,[\"User\"]%n" +
"12,,,,[\"User\"]%n");
private static final String EXPECTED_QUERY_QUOTES_NONE = String.format("a.name,a.city,a.street,labels(a)%n" +
"Andrea,Milano,Via Garibaldi, 7,[\"Address1\",\"Address\"]%n" +
"Bar Sport,,,[\"Address\"]%n" +
",,via Benni,[\"Address\"]%n");
private static final String EXPECTED_QUERY_QUOTES_ALWAYS = String.format("\"a.name\",\"a.city\",\"a.street\",\"labels(a)\"%n" +
"\"Andrea\",\"Milano\",\"Via Garibaldi, 7\",\"[\"\"Address1\"\",\"\"Address\"\"]\"%n" +
"\"Bar Sport\",\"\",\"\",\"[\"\"Address\"\"]\"%n" +
"\"\",\"\",\"via Benni\",\"[\"\"Address\"\"]\"%n");
private static final String EXPECTED_QUERY_QUOTES_NEEDED = String.format("a.name,a.city,a.street,labels(a)%n" +
"Andrea,Milano,\"Via Garibaldi, 7\",\"[\"Address1\",\"Address\"]\"%n" +
"Bar Sport,,,\"[\"Address\"]\"%n" +
",,via Benni,\"[\"Address\"]\"%n");
private static final String EXPECTED = String.format("\"_id\",\"_labels\",\"age\",\"city\",\"kids\",\"male\",\"name\",\"street\",\"_start\",\"_end\",\"_type\"%n" +
"\"0\",\":User:User1\",\"42\",\"\",\"[\"\"a\"\",\"\"b\"\",\"\"c\"\"]\",\"true\",\"foo\",\"\",,,%n" +
"\"1\",\":User\",\"42\",\"\",\"\",\"\",\"bar\",\"\",,,%n" +
Expand Down Expand Up @@ -81,113 +74,74 @@ public class ExportCsvS3Test {
",,,,,,,,0,1,KNOWS%n" +
",,,,,,,,3,4,NEXT_DELIVERY%n");

private static File directory = new File("target/import");
static { //noinspection ResultOfMethodCallIgnored
directory.mkdirs();
}

@ClassRule
public static DbmsRule db = new ImpermanentDbmsRule()
.withSetting(GraphDatabaseSettings.load_csv_file_url_root, directory.toPath().toAbsolutePath());

private static String getEnvVar(String envVarKey) throws Exception {
return Optional.ofNullable(System.getenv(envVarKey)).orElseThrow(
() -> new Exception(String.format("%s is not set in the environment", envVarKey))
);
}

@BeforeClass
public static void setUp() throws Exception {
if (S3_BUCKET_NAME == null) {
S3_BUCKET_NAME = getEnvVar("S3_BUCKET_NAME");
}
apocConfig().setProperty(APOC_EXPORT_FILE_ENABLED, true);
TestUtil.registerProcedure(db, ExportCSV.class, Graphs.class);
db.executeTransactionally("CREATE (f:User1:User {name:'foo',age:42,male:true,kids:['a','b','c']})-[:KNOWS]->(b:User {name:'bar',age:42}),(c:User {age:12})");
db.executeTransactionally("CREATE (f:Address1:Address {name:'Andrea', city: 'Milano', street:'Via Garibaldi, 7'})-[:NEXT_DELIVERY]->(a:Address {name: 'Bar Sport'}), (b:Address {street: 'via Benni'})");
}

private static String getS3Url(String key) {
return String.format("s3://:@/%s/%s", S3_BUCKET_NAME, key);
}

private String readFile(String fileName) {
return TestUtil.readFileToString(new File(directory, fileName));
}

private void verifyUpload(String s3Url, String fileName, String expected) throws IOException {
S3TestUtil.readFile(s3Url, Paths.get(directory.toString(), fileName).toString());
assertEquals(expected, readFile(fileName));
}

@Test
public void testExportAllCsvS3() throws Exception {
String fileName = "all.csv";
String s3Url = getS3Url(fileName);
String s3Url = s3Container.getUrl(fileName);
TestUtil.testCall(db, "CALL apoc.export.csv.all($s3,null)",
map("s3", s3Url),
(r) -> assertResults(s3Url, r, "database"));
verifyUpload(s3Url, fileName, EXPECTED);
assertStringFileEquals(EXPECTED, s3Url);
}

@Test
public void testExportAllCsvS3WithQuotes() throws Exception {
String fileName = "all.csv";
String s3Url = getS3Url(fileName);
String s3Url = s3Container.getUrl(fileName);
TestUtil.testCall(db, "CALL apoc.export.csv.all($s3,{quotes: true})",
map("s3", s3Url),
(r) -> assertResults(s3Url, r, "database"));
verifyUpload(s3Url, fileName, EXPECTED);
assertStringFileEquals(EXPECTED, s3Url);
}

@Test
public void testExportAllCsvS3WithoutQuotes() throws Exception {
String fileName = "all1.csv";
String s3Url = getS3Url(fileName);
String s3Url = s3Container.getUrl(fileName);
TestUtil.testCall(db, "CALL apoc.export.csv.all($s3,{quotes: 'none'})",
map("s3", s3Url),
(r) -> assertResults(s3Url, r, "database"));
verifyUpload(s3Url, fileName, EXPECTED_NONE_QUOTES);
assertStringFileEquals(EXPECTED_NONE_QUOTES, s3Url);
}

@Test
public void testExportAllCsvS3NeededQuotes() throws Exception {
String fileName = "all2.csv";
String s3Url = getS3Url(fileName);
String s3Url = s3Container.getUrl(fileName);
TestUtil.testCall(db, "CALL apoc.export.csv.all($s3,{quotes: 'ifNeeded'})",
map("s3", s3Url),
(r) -> assertResults(s3Url, r, "database"));
verifyUpload(s3Url, fileName, EXPECTED_NEEDED_QUOTES);
assertStringFileEquals(EXPECTED_NEEDED_QUOTES, s3Url);
}

@Test
public void testExportGraphCsv() throws Exception {
String fileName = "graph.csv";
String s3Url = getS3Url(fileName);
String s3Url = s3Container.getUrl(fileName);
TestUtil.testCall(db, "CALL apoc.graph.fromDB('test',{}) yield graph " +
"CALL apoc.export.csv.graph(graph, $s3,{quotes: 'none'}) " +
"YIELD nodes, relationships, properties, file, source,format, time " +
"RETURN *", map("s3", s3Url),
(r) -> assertResults(s3Url, r, "graph"));
verifyUpload(s3Url, fileName, EXPECTED_NONE_QUOTES);
assertStringFileEquals(EXPECTED_NONE_QUOTES, s3Url);
}

@Test
public void testExportGraphCsvWithoutQuotes() throws Exception {
String fileName = "graph1.csv";
String s3Url = getS3Url(fileName);
String s3Url = s3Container.getUrl(fileName);
TestUtil.testCall(db, "CALL apoc.graph.fromDB('test',{}) yield graph " +
"CALL apoc.export.csv.graph(graph, $s3,null) " +
"YIELD nodes, relationships, properties, file, source,format, time " +
"RETURN *", map("s3", s3Url),
(r) -> assertResults(s3Url, r, "graph"));
verifyUpload(s3Url, fileName, EXPECTED);
assertStringFileEquals(EXPECTED, s3Url);
}

@Test
public void testExportQueryCsv() throws Exception {
String fileName = "query.csv";
String s3Url = getS3Url(fileName);
String s3Url = s3Container.getUrl(fileName);
String query = "MATCH (u:User) return u.age, u.name, u.male, u.kids, labels(u)";
TestUtil.testCall(db, "CALL apoc.export.csv.query($query,$s3,null)",
map("s3", s3Url, "query", query),
Expand All @@ -197,13 +151,13 @@ public void testExportQueryCsv() throws Exception {
assertEquals("csv", r.get("format"));

});
verifyUpload(s3Url, fileName, EXPECTED_QUERY);
assertStringFileEquals(EXPECTED_QUERY, s3Url);
}

@Test
public void testExportQueryCsvWithoutQuotes() throws Exception {
String fileName = "query1.csv";
String s3Url = getS3Url(fileName);
String s3Url = s3Container.getUrl(fileName);
String query = "MATCH (u:User) return u.age, u.name, u.male, u.kids, labels(u)";
TestUtil.testCall(db, "CALL apoc.export.csv.query($query,$s3,{quotes: false})",
map("s3", s3Url, "query", query),
Expand All @@ -213,13 +167,13 @@ public void testExportQueryCsvWithoutQuotes() throws Exception {
assertEquals("csv", r.get("format"));

});
verifyUpload(s3Url, fileName, EXPECTED_QUERY_WITHOUT_QUOTES);
assertStringFileEquals(EXPECTED_QUERY_WITHOUT_QUOTES, s3Url);
}

@Test
public void testExportQueryNodesCsv() throws Exception {
String fileName = "query_nodes.csv";
String s3Url = getS3Url(fileName);
String s3Url = s3Container.getUrl(fileName);
String query = "MATCH (u:User) return u";
TestUtil.testCall(db, "CALL apoc.export.csv.query($query,$s3,null)",
map("s3", s3Url, "query", query),
Expand All @@ -229,13 +183,13 @@ public void testExportQueryNodesCsv() throws Exception {
assertEquals("csv", r.get("format"));

});
verifyUpload(s3Url, fileName, EXPECTED_QUERY_NODES);
assertStringFileEquals(EXPECTED_QUERY_NODES, s3Url);
}

@Test
public void testExportQueryNodesCsvParams() throws Exception {
String fileName = "query_nodes1.csv";
String s3Url = getS3Url(fileName);
String s3Url = s3Container.getUrl(fileName);
String query = "MATCH (u:User) WHERE u.age > $age return u";
TestUtil.testCall(db, "CALL apoc.export.csv.query($query,$s3,{params:{age:10}})", map("s3", s3Url, "query", query),
(r) -> {
Expand All @@ -244,7 +198,7 @@ public void testExportQueryNodesCsvParams() throws Exception {
assertEquals("csv", r.get("format"));

});
verifyUpload(s3Url, fileName, EXPECTED_QUERY_NODES);
assertStringFileEquals(EXPECTED_QUERY_NODES, s3Url);
}

private void assertResults(String fileName, Map<String, Object> r, final String source) {
Expand Down
Loading