Skip to content

Commit

Permalink
[4Bmcyc2U] Fix ignored S3 tests (neo4j/apoc#241)
Browse files Browse the repository at this point in the history 
* [4Bmcyc2U] Fix ignored S3 tests
* fix tests
* lightened ExportS3PerformanceTest
  • Loading branch information
@vga91
vga91 committed Jan 3, 2023
1 parent 343b0a0 commit 9be2d1c
Show file tree
Hide file tree
Showing 16 changed files with 699 additions and 1,675 deletions.
1 change: 0 additions & 1 deletion core/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,6 @@ dependencies {
testCompile 'net.sourceforge.jexcelapi:jxl:2.6.12'

compileOnly group: 'com.amazonaws', name: 'aws-java-sdk-s3', version: '1.12.349'
testCompile group: 'com.amazonaws', name: 'aws-java-sdk-s3', version: '1.12.349'

testCompile group: 'org.reflections', name: 'reflections', version: '0.9.12'
testCompile group: 'junit', name: 'junit', version: '4.13.1'
Expand Down
102 changes: 0 additions & 102 deletions core/src/test/java/apoc/export/ExportS3PerformanceTest.java
View file

This file was deleted.

128 changes: 39 additions & 89 deletions core/src/test/java/apoc/export/csv/ExportCsvS3Test.java
View file
Original file line number Diff line number Diff line change
@@ -1,35 +1,40 @@
package apoc.export.csv;

import apoc.ApocSettings;
import apoc.graph.Graphs;
import apoc.util.TestUtil;
import apoc.util.s3.S3TestUtil;
import org.junit.AfterClass;
import apoc.util.s3.S3BaseTest;
import org.junit.BeforeClass;
import org.junit.ClassRule;
import org.junit.Test;
import org.junit.Ignore;
import org.neo4j.configuration.GraphDatabaseSettings;
import org.neo4j.test.rule.DbmsRule;
import org.neo4j.test.rule.ImpermanentDbmsRule;

import java.io.File;
import java.io.IOException;
import java.nio.file.Paths;
import java.util.Map;
import java.util.Optional;

import static apoc.ApocConfig.APOC_EXPORT_FILE_ENABLED;
import static apoc.ApocConfig.apocConfig;
import static apoc.util.MapUtil.map;
import static apoc.util.s3.S3TestUtil.assertStringFileEquals;
import static junit.framework.TestCase.assertTrue;
import static org.junit.Assert.assertEquals;

@Ignore("To use this test, you need to set the S3 bucket and region to a valid endpoint " +
"and have your access key and secret key setup in your environment.")
public class ExportCsvS3Test {
private static String S3_BUCKET_NAME = null;
public class ExportCsvS3Test extends S3BaseTest {

@ClassRule
public static DbmsRule db = new ImpermanentDbmsRule();

@BeforeClass
public static void setUp() throws Exception{
baseBeforeClass();

apocConfig().setProperty(APOC_EXPORT_FILE_ENABLED, true);
TestUtil.registerProcedure(db, ExportCSV.class, Graphs.class);
db.executeTransactionally("CREATE (f:User1:User {name:'foo',age:42,male:true,kids:['a','b','c']})-[:KNOWS]->(b:User {name:'bar',age:42}),(c:User {age:12})");
db.executeTransactionally("CREATE (f:Address1:Address {name:'Andrea', city: 'Milano', street:'Via Garibaldi, 7'})-[:NEXT_DELIVERY]->(a:Address {name: 'Bar Sport'}), (b:Address {street: 'via Benni'})");
}

private static final String EXPECTED_QUERY_NODES = String.format("\"u\"%n" +
"\"{\"\"id\"\":0,\"\"labels\"\":[\"\"User\"\",\"\"User1\"\"],\"\"properties\"\":{\"\"name\"\":\"\"foo\"\",\"\"age\"\":42,\"\"male\"\":true,\"\"kids\"\":[\"\"a\"\",\"\"b\"\",\"\"c\"\"]}}\"%n" +
"\"{\"\"id\"\":0,\"\"labels\"\":[\"\"User\"\",\"\"User1\"\"],\"\"properties\"\":{\"\"name\"\":\"\"foo\"\",\"\"male\"\":true,\"\"age\"\":42,\"\"kids\"\":[\"\"a\"\",\"\"b\"\",\"\"c\"\"]}}\"%n" +
"\"{\"\"id\"\":1,\"\"labels\"\":[\"\"User\"\"],\"\"properties\"\":{\"\"name\"\":\"\"bar\"\",\"\"age\"\":42}}\"%n" +
"\"{\"\"id\"\":2,\"\"labels\"\":[\"\"User\"\"],\"\"properties\"\":{\"\"age\"\":12}}\"%n");
private static final String EXPECTED_QUERY = String.format("\"u.age\",\"u.name\",\"u.male\",\"u.kids\",\"labels(u)\"%n" +
Expand All @@ -40,18 +45,6 @@ public class ExportCsvS3Test {
"42,foo,true,[\"a\",\"b\",\"c\"],[\"User1\",\"User\"]%n" +
"42,bar,,,[\"User\"]%n" +
"12,,,,[\"User\"]%n");
private static final String EXPECTED_QUERY_QUOTES_NONE = String.format("a.name,a.city,a.street,labels(a)%n" +
"Andrea,Milano,Via Garibaldi, 7,[\"Address1\",\"Address\"]%n" +
"Bar Sport,,,[\"Address\"]%n" +
",,via Benni,[\"Address\"]%n");
private static final String EXPECTED_QUERY_QUOTES_ALWAYS = String.format("\"a.name\",\"a.city\",\"a.street\",\"labels(a)\"%n" +
"\"Andrea\",\"Milano\",\"Via Garibaldi, 7\",\"[\"\"Address1\"\",\"\"Address\"\"]\"%n" +
"\"Bar Sport\",\"\",\"\",\"[\"\"Address\"\"]\"%n" +
"\"\",\"\",\"via Benni\",\"[\"\"Address\"\"]\"%n");
private static final String EXPECTED_QUERY_QUOTES_NEEDED = String.format("a.name,a.city,a.street,labels(a)%n" +
"Andrea,Milano,\"Via Garibaldi, 7\",\"[\"Address1\",\"Address\"]\"%n" +
"Bar Sport,,,\"[\"Address\"]\"%n" +
",,via Benni,\"[\"Address\"]\"%n");
private static final String EXPECTED = String.format("\"_id\",\"_labels\",\"age\",\"city\",\"kids\",\"male\",\"name\",\"street\",\"_start\",\"_end\",\"_type\"%n" +
"\"0\",\":User:User1\",\"42\",\"\",\"[\"\"a\"\",\"\"b\"\",\"\"c\"\"]\",\"true\",\"foo\",\"\",,,%n" +
"\"1\",\":User\",\"42\",\"\",\"\",\"\",\"bar\",\"\",,,%n" +
Expand Down Expand Up @@ -81,117 +74,74 @@ public class ExportCsvS3Test {
",,,,,,,,0,1,KNOWS%n" +
",,,,,,,,3,4,NEXT_DELIVERY%n");

private static File directory = new File("target/import");
static { //noinspection ResultOfMethodCallIgnored
directory.mkdirs();
}

@ClassRule
public static DbmsRule db = new ImpermanentDbmsRule()
.withSetting(GraphDatabaseSettings.load_csv_file_url_root, directory.toPath().toAbsolutePath())
.withSetting(ApocSettings.apoc_export_file_enabled, true);

private static String getEnvVar(String envVarKey) throws Exception {
return Optional.ofNullable(System.getenv(envVarKey)).orElseThrow(
() -> new Exception(String.format("%s is not set in the environment", envVarKey))
);
}

@BeforeClass
public static void setUp() throws Exception {
if (S3_BUCKET_NAME == null) {
S3_BUCKET_NAME = getEnvVar("S3_BUCKET_NAME");
}
TestUtil.registerProcedure(db, ExportCSV.class, Graphs.class);
db.executeTransactionally("CREATE (f:User1:User {name:'foo',age:42,male:true,kids:['a','b','c']})-[:KNOWS]->(b:User {name:'bar',age:42}),(c:User {age:12})");
db.executeTransactionally("CREATE (f:Address1:Address {name:'Andrea', city: 'Milano', street:'Via Garibaldi, 7'})-[:NEXT_DELIVERY]->(a:Address {name: 'Bar Sport'}), (b:Address {street: 'via Benni'})");
}

@AfterClass
public static void tearDown() {
}

private static String getS3Url(String key) {
return String.format("s3://:@/%s/%s", S3_BUCKET_NAME, key);
}

private String readFile(String fileName) {
return TestUtil.readFileToString(new File(directory, fileName));
}

private void verifyUpload(String s3Url, String fileName, String expected) throws IOException {
S3TestUtil.readFile(s3Url, Paths.get(directory.toString(), fileName).toString());
assertEquals(expected, readFile(fileName));
}

@Test
public void testExportAllCsvS3() throws Exception {
String fileName = "all.csv";
String s3Url = getS3Url(fileName);
String s3Url = s3Container.getUrl(fileName);
TestUtil.testCall(db, "CALL apoc.export.csv.all($s3,null)",
map("s3", s3Url),
(r) -> assertResults(s3Url, r, "database"));
verifyUpload(s3Url, fileName, EXPECTED);
assertStringFileEquals(EXPECTED, s3Url);
}

@Test
public void testExportAllCsvS3WithQuotes() throws Exception {
String fileName = "all.csv";
String s3Url = getS3Url(fileName);
String s3Url = s3Container.getUrl(fileName);
TestUtil.testCall(db, "CALL apoc.export.csv.all($s3,{quotes: true})",
map("s3", s3Url),
(r) -> assertResults(s3Url, r, "database"));
verifyUpload(s3Url, fileName, EXPECTED);
assertStringFileEquals(EXPECTED, s3Url);
}

@Test
public void testExportAllCsvS3WithoutQuotes() throws Exception {
String fileName = "all1.csv";
String s3Url = getS3Url(fileName);
String s3Url = s3Container.getUrl(fileName);
TestUtil.testCall(db, "CALL apoc.export.csv.all($s3,{quotes: 'none'})",
map("s3", s3Url),
(r) -> assertResults(s3Url, r, "database"));
verifyUpload(s3Url, fileName, EXPECTED_NONE_QUOTES);
assertStringFileEquals(EXPECTED_NONE_QUOTES, s3Url);
}

@Test
public void testExportAllCsvS3NeededQuotes() throws Exception {
String fileName = "all2.csv";
String s3Url = getS3Url(fileName);
String s3Url = s3Container.getUrl(fileName);
TestUtil.testCall(db, "CALL apoc.export.csv.all($s3,{quotes: 'ifNeeded'})",
map("s3", s3Url),
(r) -> assertResults(s3Url, r, "database"));
verifyUpload(s3Url, fileName, EXPECTED_NEEDED_QUOTES);
assertStringFileEquals(EXPECTED_NEEDED_QUOTES, s3Url);
}

@Test
public void testExportGraphCsv() throws Exception {
String fileName = "graph.csv";
String s3Url = getS3Url(fileName);
String s3Url = s3Container.getUrl(fileName);
TestUtil.testCall(db, "CALL apoc.graph.fromDB('test',{}) yield graph " +
"CALL apoc.export.csv.graph(graph, $s3,{quotes: 'none'}) " +
"YIELD nodes, relationships, properties, file, source,format, time " +
"RETURN *", map("s3", s3Url),
(r) -> assertResults(s3Url, r, "graph"));
verifyUpload(s3Url, fileName, EXPECTED_NONE_QUOTES);
assertStringFileEquals(EXPECTED_NONE_QUOTES, s3Url);
}

@Test
public void testExportGraphCsvWithoutQuotes() throws Exception {
String fileName = "graph1.csv";
String s3Url = getS3Url(fileName);
String s3Url = s3Container.getUrl(fileName);
TestUtil.testCall(db, "CALL apoc.graph.fromDB('test',{}) yield graph " +
"CALL apoc.export.csv.graph(graph, $s3,null) " +
"YIELD nodes, relationships, properties, file, source,format, time " +
"RETURN *", map("s3", s3Url),
(r) -> assertResults(s3Url, r, "graph"));
verifyUpload(s3Url, fileName, EXPECTED);
assertStringFileEquals(EXPECTED, s3Url);
}

@Test
public void testExportQueryCsv() throws Exception {
String fileName = "query.csv";
String s3Url = getS3Url(fileName);
String s3Url = s3Container.getUrl(fileName);
String query = "MATCH (u:User) return u.age, u.name, u.male, u.kids, labels(u)";
TestUtil.testCall(db, "CALL apoc.export.csv.query($query,$s3,null)",
map("s3", s3Url, "query", query),
Expand All @@ -201,13 +151,13 @@ public void testExportQueryCsv() throws Exception {
assertEquals("csv", r.get("format"));

});
verifyUpload(s3Url, fileName, EXPECTED_QUERY);
assertStringFileEquals(EXPECTED_QUERY, s3Url);
}

@Test
public void testExportQueryCsvWithoutQuotes() throws Exception {
String fileName = "query1.csv";
String s3Url = getS3Url(fileName);
String s3Url = s3Container.getUrl(fileName);
String query = "MATCH (u:User) return u.age, u.name, u.male, u.kids, labels(u)";
TestUtil.testCall(db, "CALL apoc.export.csv.query($query,$s3,{quotes: false})",
map("s3", s3Url, "query", query),
Expand All @@ -217,13 +167,13 @@ public void testExportQueryCsvWithoutQuotes() throws Exception {
assertEquals("csv", r.get("format"));

});
verifyUpload(s3Url, fileName, EXPECTED_QUERY_WITHOUT_QUOTES);
assertStringFileEquals(EXPECTED_QUERY_WITHOUT_QUOTES, s3Url);
}

@Test
public void testExportQueryNodesCsv() throws Exception {
String fileName = "query_nodes.csv";
String s3Url = getS3Url(fileName);
String s3Url = s3Container.getUrl(fileName);
String query = "MATCH (u:User) return u";
TestUtil.testCall(db, "CALL apoc.export.csv.query($query,$s3,null)",
map("s3", s3Url, "query", query),
Expand All @@ -233,13 +183,13 @@ public void testExportQueryNodesCsv() throws Exception {
assertEquals("csv", r.get("format"));

});
verifyUpload(s3Url, fileName, EXPECTED_QUERY_NODES);
assertStringFileEquals(EXPECTED_QUERY_NODES, s3Url);
}

@Test
public void testExportQueryNodesCsvParams() throws Exception {
String fileName = "query_nodes1.csv";
String s3Url = getS3Url(fileName);
String s3Url = s3Container.getUrl(fileName);
String query = "MATCH (u:User) WHERE u.age > $age return u";
TestUtil.testCall(db, "CALL apoc.export.csv.query($query,$s3,{params:{age:10}})", map("s3", s3Url, "query", query),
(r) -> {
Expand All @@ -248,7 +198,7 @@ public void testExportQueryNodesCsvParams() throws Exception {
assertEquals("csv", r.get("format"));

});
verifyUpload(s3Url, fileName, EXPECTED_QUERY_NODES);
assertStringFileEquals(EXPECTED_QUERY_NODES, s3Url);
}

private void assertResults(String fileName, Map<String, Object> r, final String source) {
Expand Down
Loading

0 comments on commit 9be2d1c

Please sign in to comment.