From 43a0a1135d3ac1087c09ad7e1eaba3a29ae6a21f Mon Sep 17 00:00:00 2001 From: Giuseppe Villani Date: Thu, 17 Nov 2022 09:29:58 +0100 Subject: [PATCH 1/3] [4Bmcyc2U] Fix ignored S3 tests --- common/build.gradle | 1 - .../test/java/apoc/util/s3/S3TestUtil.java | 32 --- .../apoc/export/ExportS3PerformanceTest.java | 52 +---- .../java/apoc/export/csv/ExportCsvS3Test.java | 118 ++++------ .../export/cypher/ExportCypherS3Test.java | 198 ++++++++--------- .../export/graphml/ExportGraphMLS3Test.java | 204 +++--------------- .../export/graphml/ExportGraphMLTest.java | 186 ++-------------- .../export/graphml/ExportGraphMLTestUtil.java | 188 ++++++++++++++++ .../apoc/export/json/ExportJsonS3Test.java | 157 +++++--------- .../java/apoc/export/json/ExportJsonTest.java | 12 +- test-utils/build.gradle | 1 + .../src/main/java/apoc/util/FileTestUtil.java | 18 ++ .../main/java/apoc/util/s3/S3BaseTest.java | 25 +++ .../main/java/apoc/util/s3/S3Container.java | 65 ++++++ .../apoc/util/s3/S3ParamsExtractorTest.java | 0 .../main/java/apoc/util/s3/S3TestUtil.java | 47 ++++ 16 files changed, 581 insertions(+), 723 deletions(-) delete mode 100644 common/src/test/java/apoc/util/s3/S3TestUtil.java create mode 100644 core/src/test/java/apoc/export/graphml/ExportGraphMLTestUtil.java create mode 100644 test-utils/src/main/java/apoc/util/FileTestUtil.java create mode 100644 test-utils/src/main/java/apoc/util/s3/S3BaseTest.java create mode 100644 test-utils/src/main/java/apoc/util/s3/S3Container.java rename {common/src/test => test-utils/src/main}/java/apoc/util/s3/S3ParamsExtractorTest.java (100%) create mode 100644 test-utils/src/main/java/apoc/util/s3/S3TestUtil.java diff --git a/common/build.gradle b/common/build.gradle index d94d6a566..97f290464 100644 --- a/common/build.gradle +++ b/common/build.gradle @@ -74,7 +74,6 @@ dependencies { // These dependencies affect the tests only, they will not be packaged in the resulting .jar testImplementation project(':test-utils') - testImplementation group: 'com.amazonaws', name: 'aws-java-sdk-s3', version: '1.12.348' testImplementation group: 'junit', name: 'junit', version: '4.13.2' testImplementation group: 'com.github.stefanbirkner', name: 'system-rules', version: '1.19.0' testImplementation group: 'org.hamcrest', name: 'hamcrest-library', version: '1.3' diff --git a/common/src/test/java/apoc/util/s3/S3TestUtil.java b/common/src/test/java/apoc/util/s3/S3TestUtil.java deleted file mode 100644 index ef45d5176..000000000 --- a/common/src/test/java/apoc/util/s3/S3TestUtil.java +++ /dev/null @@ -1,32 +0,0 @@ -package apoc.util.s3; - -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.model.S3Object; -import com.amazonaws.services.s3.model.S3ObjectInputStream; -import org.apache.commons.io.FileUtils; - -import java.io.File; -import java.io.IOException; -import java.net.URL; - -/** - * Utility class for testing Amazon S3 related functionality. - */ -public class S3TestUtil { - - /** - * Read object from S3 bucket and save it as a file. This code expects valid AWS credentials are set up. - * @param s3Url String containing url to S3 bucket. - * @param pathname Local pathname where the file will be stored. - * @throws IOException Exception thrown if copying stream to file fails. - */ - public static void readFile(String s3Url, String pathname) throws IOException { - S3Params s3Params = S3ParamsExtractor.extract(new URL(s3Url)); - S3Aws s3Aws = new S3Aws(s3Params, s3Params.getRegion()); - AmazonS3 s3Client = s3Aws.getClient(); - - S3Object s3object = s3Client.getObject(s3Params.getBucket(), s3Params.getKey()); - S3ObjectInputStream inputStream = s3object.getObjectContent(); - FileUtils.copyInputStreamToFile(inputStream, new File(pathname)); - } -} diff --git a/core/src/test/java/apoc/export/ExportS3PerformanceTest.java b/core/src/test/java/apoc/export/ExportS3PerformanceTest.java index a31922755..dbc77840e 100644 --- a/core/src/test/java/apoc/export/ExportS3PerformanceTest.java +++ b/core/src/test/java/apoc/export/ExportS3PerformanceTest.java @@ -3,60 +3,31 @@ import apoc.export.csv.ExportCSV; import apoc.graph.Graphs; import apoc.util.TestUtil; -import apoc.util.s3.S3Aws; -import apoc.util.s3.S3Params; -import apoc.util.s3.S3ParamsExtractor; -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.model.ObjectListing; -import com.amazonaws.services.s3.model.S3ObjectSummary; +import apoc.util.s3.S3BaseTest; +import com.amazonaws.services.s3.model.S3Object; import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.Test; -import org.junit.Ignore; import org.neo4j.test.rule.DbmsRule; import org.neo4j.test.rule.ImpermanentDbmsRule; import java.io.IOException; -import java.net.URL; import java.time.Duration; import java.time.Instant; -import java.util.Optional; import static apoc.ApocConfig.APOC_EXPORT_FILE_ENABLED; import static apoc.ApocConfig.apocConfig; import static apoc.util.MapUtil.map; -import static junit.framework.TestCase.assertTrue; +import static apoc.util.s3.S3TestUtil.getS3Object; +import static junit.framework.TestCase.assertEquals; -@Ignore("To use this test, you need to set the S3 bucket and region to a valid endpoint " + - "and have your access key and secret key setup in your environment.") -public class ExportS3PerformanceTest { - private static String S3_BUCKET_NAME = null; - private static int REPEAT_TEST = 3; - private static String getEnvVar(String envVarKey) throws Exception { - return Optional.ofNullable(System.getenv(envVarKey)).orElseThrow( - () -> new Exception(String.format("%s is not set in the environment", envVarKey)) - ); - } - - private static String getS3Url(String key) { - return String.format("s3://:/%s/%s", S3_BUCKET_NAME, key); - } +public class ExportS3PerformanceTest extends S3BaseTest { + private final static int REPEAT_TEST = 3; private void verifyFileUploaded(String s3Url, String fileName) throws IOException { - S3Params s3Params = S3ParamsExtractor.extract(new URL(s3Url)); - S3Aws s3Aws = new S3Aws(s3Params, s3Params.getRegion()); - AmazonS3 s3Client = s3Aws.getClient(); - - boolean fileUploaded = false; - ObjectListing objectListing = s3Client.listObjects(s3Params.getBucket()); - for(S3ObjectSummary os : objectListing.getObjectSummaries()) { - if (os.getKey().equals(fileName)) { - fileUploaded = true; - break; - } - } - assertTrue(fileUploaded); + final S3Object s3Object = getS3Object(s3Url); + assertEquals(fileName, s3Object.getKey()); } @ClassRule @@ -64,9 +35,8 @@ private void verifyFileUploaded(String s3Url, String fileName) throws IOExceptio @BeforeClass public static void setUp() throws Exception { - if (S3_BUCKET_NAME == null) { - S3_BUCKET_NAME = getEnvVar("S3_BUCKET_NAME"); - } + baseBeforeClass(); + TestUtil.registerProcedure(db, ExportCSV.class, Graphs.class); apocConfig().setProperty(APOC_EXPORT_FILE_ENABLED, true); @@ -85,7 +55,7 @@ public void testExportAllCsvS3() throws Exception { System.out.println("Test started."); for (int repeat=1; repeat<=REPEAT_TEST; repeat++) { String fileName = String.format("performanceTest_%d.csv", repeat); - String s3Url = getS3Url(fileName); + String s3Url = s3Container.getUrl(fileName); // Run the performance testing final Instant start = Instant.now(); diff --git a/core/src/test/java/apoc/export/csv/ExportCsvS3Test.java b/core/src/test/java/apoc/export/csv/ExportCsvS3Test.java index fb854f051..8fd3f05f4 100644 --- a/core/src/test/java/apoc/export/csv/ExportCsvS3Test.java +++ b/core/src/test/java/apoc/export/csv/ExportCsvS3Test.java @@ -2,31 +2,36 @@ import apoc.graph.Graphs; import apoc.util.TestUtil; -import apoc.util.s3.S3TestUtil; +import apoc.util.s3.S3BaseTest; import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.Test; -import org.junit.Ignore; -import org.neo4j.configuration.GraphDatabaseSettings; import org.neo4j.test.rule.DbmsRule; import org.neo4j.test.rule.ImpermanentDbmsRule; -import java.io.File; -import java.io.IOException; -import java.nio.file.Paths; import java.util.Map; -import java.util.Optional; import static apoc.ApocConfig.APOC_EXPORT_FILE_ENABLED; import static apoc.ApocConfig.apocConfig; import static apoc.util.MapUtil.map; +import static apoc.util.s3.S3TestUtil.assertStringFileEquals; import static junit.framework.TestCase.assertTrue; import static org.junit.Assert.assertEquals; -@Ignore("To use this test, you need to set the S3 bucket and region to a valid endpoint " + - "and have your access key and secret key setup in your environment.") -public class ExportCsvS3Test { - private static String S3_BUCKET_NAME = null; +public class ExportCsvS3Test extends S3BaseTest { + + @ClassRule + public static DbmsRule db = new ImpermanentDbmsRule(); + + @BeforeClass + public static void setUp() throws Exception{ + baseBeforeClass(); + + apocConfig().setProperty(APOC_EXPORT_FILE_ENABLED, true); + TestUtil.registerProcedure(db, ExportCSV.class, Graphs.class); + db.executeTransactionally("CREATE (f:User1:User {name:'foo',age:42,male:true,kids:['a','b','c']})-[:KNOWS]->(b:User {name:'bar',age:42}),(c:User {age:12})"); + db.executeTransactionally("CREATE (f:Address1:Address {name:'Andrea', city: 'Milano', street:'Via Garibaldi, 7'})-[:NEXT_DELIVERY]->(a:Address {name: 'Bar Sport'}), (b:Address {street: 'via Benni'})"); + } private static final String EXPECTED_QUERY_NODES = String.format("\"u\"%n" + "\"{\"\"id\"\":0,\"\"labels\"\":[\"\"User\"\",\"\"User1\"\"],\"\"properties\"\":{\"\"name\"\":\"\"foo\"\",\"\"age\"\":42,\"\"male\"\":true,\"\"kids\"\":[\"\"a\"\",\"\"b\"\",\"\"c\"\"]}}\"%n" + @@ -40,18 +45,6 @@ public class ExportCsvS3Test { "42,foo,true,[\"a\",\"b\",\"c\"],[\"User1\",\"User\"]%n" + "42,bar,,,[\"User\"]%n" + "12,,,,[\"User\"]%n"); - private static final String EXPECTED_QUERY_QUOTES_NONE = String.format("a.name,a.city,a.street,labels(a)%n" + - "Andrea,Milano,Via Garibaldi, 7,[\"Address1\",\"Address\"]%n" + - "Bar Sport,,,[\"Address\"]%n" + - ",,via Benni,[\"Address\"]%n"); - private static final String EXPECTED_QUERY_QUOTES_ALWAYS = String.format("\"a.name\",\"a.city\",\"a.street\",\"labels(a)\"%n" + - "\"Andrea\",\"Milano\",\"Via Garibaldi, 7\",\"[\"\"Address1\"\",\"\"Address\"\"]\"%n" + - "\"Bar Sport\",\"\",\"\",\"[\"\"Address\"\"]\"%n" + - "\"\",\"\",\"via Benni\",\"[\"\"Address\"\"]\"%n"); - private static final String EXPECTED_QUERY_QUOTES_NEEDED = String.format("a.name,a.city,a.street,labels(a)%n" + - "Andrea,Milano,\"Via Garibaldi, 7\",\"[\"Address1\",\"Address\"]\"%n" + - "Bar Sport,,,\"[\"Address\"]\"%n" + - ",,via Benni,\"[\"Address\"]\"%n"); private static final String EXPECTED = String.format("\"_id\",\"_labels\",\"age\",\"city\",\"kids\",\"male\",\"name\",\"street\",\"_start\",\"_end\",\"_type\"%n" + "\"0\",\":User:User1\",\"42\",\"\",\"[\"\"a\"\",\"\"b\"\",\"\"c\"\"]\",\"true\",\"foo\",\"\",,,%n" + "\"1\",\":User\",\"42\",\"\",\"\",\"\",\"bar\",\"\",,,%n" + @@ -81,113 +74,74 @@ public class ExportCsvS3Test { ",,,,,,,,0,1,KNOWS%n" + ",,,,,,,,3,4,NEXT_DELIVERY%n"); - private static File directory = new File("target/import"); - static { //noinspection ResultOfMethodCallIgnored - directory.mkdirs(); - } - - @ClassRule - public static DbmsRule db = new ImpermanentDbmsRule() - .withSetting(GraphDatabaseSettings.load_csv_file_url_root, directory.toPath().toAbsolutePath()); - - private static String getEnvVar(String envVarKey) throws Exception { - return Optional.ofNullable(System.getenv(envVarKey)).orElseThrow( - () -> new Exception(String.format("%s is not set in the environment", envVarKey)) - ); - } - - @BeforeClass - public static void setUp() throws Exception { - if (S3_BUCKET_NAME == null) { - S3_BUCKET_NAME = getEnvVar("S3_BUCKET_NAME"); - } - apocConfig().setProperty(APOC_EXPORT_FILE_ENABLED, true); - TestUtil.registerProcedure(db, ExportCSV.class, Graphs.class); - db.executeTransactionally("CREATE (f:User1:User {name:'foo',age:42,male:true,kids:['a','b','c']})-[:KNOWS]->(b:User {name:'bar',age:42}),(c:User {age:12})"); - db.executeTransactionally("CREATE (f:Address1:Address {name:'Andrea', city: 'Milano', street:'Via Garibaldi, 7'})-[:NEXT_DELIVERY]->(a:Address {name: 'Bar Sport'}), (b:Address {street: 'via Benni'})"); - } - - private static String getS3Url(String key) { - return String.format("s3://:@/%s/%s", S3_BUCKET_NAME, key); - } - - private String readFile(String fileName) { - return TestUtil.readFileToString(new File(directory, fileName)); - } - - private void verifyUpload(String s3Url, String fileName, String expected) throws IOException { - S3TestUtil.readFile(s3Url, Paths.get(directory.toString(), fileName).toString()); - assertEquals(expected, readFile(fileName)); - } - @Test public void testExportAllCsvS3() throws Exception { String fileName = "all.csv"; - String s3Url = getS3Url(fileName); + String s3Url = s3Container.getUrl(fileName); TestUtil.testCall(db, "CALL apoc.export.csv.all($s3,null)", map("s3", s3Url), (r) -> assertResults(s3Url, r, "database")); - verifyUpload(s3Url, fileName, EXPECTED); + assertStringFileEquals(EXPECTED, s3Url); } @Test public void testExportAllCsvS3WithQuotes() throws Exception { String fileName = "all.csv"; - String s3Url = getS3Url(fileName); + String s3Url = s3Container.getUrl(fileName); TestUtil.testCall(db, "CALL apoc.export.csv.all($s3,{quotes: true})", map("s3", s3Url), (r) -> assertResults(s3Url, r, "database")); - verifyUpload(s3Url, fileName, EXPECTED); + assertStringFileEquals(EXPECTED, s3Url); } @Test public void testExportAllCsvS3WithoutQuotes() throws Exception { String fileName = "all1.csv"; - String s3Url = getS3Url(fileName); + String s3Url = s3Container.getUrl(fileName); TestUtil.testCall(db, "CALL apoc.export.csv.all($s3,{quotes: 'none'})", map("s3", s3Url), (r) -> assertResults(s3Url, r, "database")); - verifyUpload(s3Url, fileName, EXPECTED_NONE_QUOTES); + assertStringFileEquals(EXPECTED_NONE_QUOTES, s3Url); } @Test public void testExportAllCsvS3NeededQuotes() throws Exception { String fileName = "all2.csv"; - String s3Url = getS3Url(fileName); + String s3Url = s3Container.getUrl(fileName); TestUtil.testCall(db, "CALL apoc.export.csv.all($s3,{quotes: 'ifNeeded'})", map("s3", s3Url), (r) -> assertResults(s3Url, r, "database")); - verifyUpload(s3Url, fileName, EXPECTED_NEEDED_QUOTES); + assertStringFileEquals(EXPECTED_NEEDED_QUOTES, s3Url); } @Test public void testExportGraphCsv() throws Exception { String fileName = "graph.csv"; - String s3Url = getS3Url(fileName); + String s3Url = s3Container.getUrl(fileName); TestUtil.testCall(db, "CALL apoc.graph.fromDB('test',{}) yield graph " + "CALL apoc.export.csv.graph(graph, $s3,{quotes: 'none'}) " + "YIELD nodes, relationships, properties, file, source,format, time " + "RETURN *", map("s3", s3Url), (r) -> assertResults(s3Url, r, "graph")); - verifyUpload(s3Url, fileName, EXPECTED_NONE_QUOTES); + assertStringFileEquals(EXPECTED_NONE_QUOTES, s3Url); } @Test public void testExportGraphCsvWithoutQuotes() throws Exception { String fileName = "graph1.csv"; - String s3Url = getS3Url(fileName); + String s3Url = s3Container.getUrl(fileName); TestUtil.testCall(db, "CALL apoc.graph.fromDB('test',{}) yield graph " + "CALL apoc.export.csv.graph(graph, $s3,null) " + "YIELD nodes, relationships, properties, file, source,format, time " + "RETURN *", map("s3", s3Url), (r) -> assertResults(s3Url, r, "graph")); - verifyUpload(s3Url, fileName, EXPECTED); + assertStringFileEquals(EXPECTED, s3Url); } @Test public void testExportQueryCsv() throws Exception { String fileName = "query.csv"; - String s3Url = getS3Url(fileName); + String s3Url = s3Container.getUrl(fileName); String query = "MATCH (u:User) return u.age, u.name, u.male, u.kids, labels(u)"; TestUtil.testCall(db, "CALL apoc.export.csv.query($query,$s3,null)", map("s3", s3Url, "query", query), @@ -197,13 +151,13 @@ public void testExportQueryCsv() throws Exception { assertEquals("csv", r.get("format")); }); - verifyUpload(s3Url, fileName, EXPECTED_QUERY); + assertStringFileEquals(EXPECTED_QUERY, s3Url); } @Test public void testExportQueryCsvWithoutQuotes() throws Exception { String fileName = "query1.csv"; - String s3Url = getS3Url(fileName); + String s3Url = s3Container.getUrl(fileName); String query = "MATCH (u:User) return u.age, u.name, u.male, u.kids, labels(u)"; TestUtil.testCall(db, "CALL apoc.export.csv.query($query,$s3,{quotes: false})", map("s3", s3Url, "query", query), @@ -213,13 +167,13 @@ public void testExportQueryCsvWithoutQuotes() throws Exception { assertEquals("csv", r.get("format")); }); - verifyUpload(s3Url, fileName, EXPECTED_QUERY_WITHOUT_QUOTES); + assertStringFileEquals(EXPECTED_QUERY_WITHOUT_QUOTES, s3Url); } @Test public void testExportQueryNodesCsv() throws Exception { String fileName = "query_nodes.csv"; - String s3Url = getS3Url(fileName); + String s3Url = s3Container.getUrl(fileName); String query = "MATCH (u:User) return u"; TestUtil.testCall(db, "CALL apoc.export.csv.query($query,$s3,null)", map("s3", s3Url, "query", query), @@ -229,13 +183,13 @@ public void testExportQueryNodesCsv() throws Exception { assertEquals("csv", r.get("format")); }); - verifyUpload(s3Url, fileName, EXPECTED_QUERY_NODES); + assertStringFileEquals(EXPECTED_QUERY_NODES, s3Url); } @Test public void testExportQueryNodesCsvParams() throws Exception { String fileName = "query_nodes1.csv"; - String s3Url = getS3Url(fileName); + String s3Url = s3Container.getUrl(fileName); String query = "MATCH (u:User) WHERE u.age > $age return u"; TestUtil.testCall(db, "CALL apoc.export.csv.query($query,$s3,{params:{age:10}})", map("s3", s3Url, "query", query), (r) -> { @@ -244,7 +198,7 @@ public void testExportQueryNodesCsvParams() throws Exception { assertEquals("csv", r.get("format")); }); - verifyUpload(s3Url, fileName, EXPECTED_QUERY_NODES); + assertStringFileEquals(EXPECTED_QUERY_NODES, s3Url); } private void assertResults(String fileName, Map r, final String source) { diff --git a/core/src/test/java/apoc/export/cypher/ExportCypherS3Test.java b/core/src/test/java/apoc/export/cypher/ExportCypherS3Test.java index 1e860e094..b904c6287 100644 --- a/core/src/test/java/apoc/export/cypher/ExportCypherS3Test.java +++ b/core/src/test/java/apoc/export/cypher/ExportCypherS3Test.java @@ -2,45 +2,32 @@ import apoc.graph.Graphs; import apoc.util.TestUtil; +import apoc.util.s3.S3BaseTest; import apoc.util.s3.S3TestUtil; import org.junit.Before; import org.junit.Rule; import org.junit.Test; -import org.junit.Ignore; import org.junit.rules.TestName; -import org.neo4j.configuration.GraphDatabaseSettings; import org.neo4j.test.rule.DbmsRule; import org.neo4j.test.rule.ImpermanentDbmsRule; -import java.io.File; import java.io.IOException; -import java.nio.file.Paths; import java.util.Map; -import java.util.Optional; import static apoc.ApocConfig.APOC_EXPORT_FILE_ENABLED; import static apoc.ApocConfig.apocConfig; import static apoc.export.cypher.ExportCypherTest.ExportCypherResults.*; import static apoc.export.util.ExportFormat.*; import static apoc.util.Util.map; +import static apoc.util.s3.S3TestUtil.assertStringFileEquals; import static org.junit.Assert.*; -@Ignore("To use this test, you need to set the S3 bucket and region to a valid endpoint " + - "and have your access key and secret key setup in your environment.") -public class ExportCypherS3Test { - private static String S3_BUCKET_NAME = null; +public class ExportCypherS3Test extends S3BaseTest { private static final Map exportConfig = map("useOptimizations", map("type", "none"), "separateFiles", true, "format", "neo4j-admin"); - private static File directory = new File("target/import"); - - static { //noinspection ResultOfMethodCallIgnored - directory.mkdirs(); - } - @Rule - public DbmsRule db = new ImpermanentDbmsRule() - .withSetting(GraphDatabaseSettings.load_csv_file_url_root, directory.toPath().toAbsolutePath()); + public DbmsRule db = new ImpermanentDbmsRule(); @Rule public TestName testName = new TestName(); @@ -48,31 +35,9 @@ public class ExportCypherS3Test { private static final String OPTIMIZED = "Optimized"; private static final String ODD = "OddDataset"; - private static String getS3Url(String key) { - return String.format("s3://:@/%s/%s", S3_BUCKET_NAME, key); - } - - private void verifyUpload(String fileName, String expected) throws IOException { - String s3Url = getS3Url(fileName); - S3TestUtil.readFile(s3Url, Paths.get(directory.toString(), fileName).toString()); - assertEquals(expected, readFile(fileName)); - } - - private static String readFile(String fileName) { - return TestUtil.readFileToString(new File(directory, fileName)); - } - - private static String getEnvVar(String envVarKey) throws Exception { - return Optional.ofNullable(System.getenv(envVarKey)).orElseThrow( - () -> new Exception(String.format("%s is not set in the environment", envVarKey)) - ); - } @Before public void setUp() throws Exception { - if (S3_BUCKET_NAME == null) { - S3_BUCKET_NAME = getEnvVar("S3_BUCKET_NAME"); - } apocConfig().setProperty(APOC_EXPORT_FILE_ENABLED, true); TestUtil.registerProcedure(db, ExportCypher.class, Graphs.class); db.executeTransactionally("CREATE RANGE INDEX FOR (n:Bar) ON (n.first_name, n.last_name)"); @@ -99,138 +64,138 @@ public void setUp() throws Exception { @Test public void testExportAllCypherDefault() throws Exception { String fileName = "all.cypher"; - String s3Url = getS3Url(fileName); + String s3Url = s3Container.getUrl(fileName); TestUtil.testCall(db, "CALL apoc.export.cypher.all($s3,{useOptimizations: { type: 'none'}, format: 'neo4j-shell'})", map("s3", s3Url), (r) -> assertResults(s3Url, r, "database")); - verifyUpload(fileName, EXPECTED_NEO4J_SHELL); + assertStringFileEquals(EXPECTED_NEO4J_SHELL, s3Url); } @Test public void testExportAllCypherForCypherShell() throws Exception { String fileName = "all.cypher"; - String s3Url = getS3Url(fileName); + String s3Url = s3Container.getUrl(fileName); TestUtil.testCall(db, "CALL apoc.export.cypher.all($s3,$config)", map("s3", s3Url, "config", map("useOptimizations", map("type", "none"), "format", "cypher-shell")), (r) -> assertResults(s3Url, r, "database")); - verifyUpload(fileName, EXPECTED_CYPHER_SHELL); + assertStringFileEquals(EXPECTED_CYPHER_SHELL, s3Url); } @Test public void testExportQueryCypherForNeo4j() throws Exception { String fileName = "all.cypher"; - String s3Url = getS3Url(fileName); + String s3Url = s3Container.getUrl(fileName); String query = "MATCH (n) OPTIONAL MATCH p = (n)-[r]-(m) RETURN n,r,m"; TestUtil.testCall(db, "CALL apoc.export.cypher.query($query,$s3,$config)", map("s3", s3Url, "query", query, "config", map("useOptimizations", map("type", "none"), "format", "neo4j-shell")), (r) -> { }); - verifyUpload(fileName, EXPECTED_NEO4J_SHELL); + assertStringFileEquals(EXPECTED_NEO4J_SHELL, s3Url); } @Test public void testExportGraphCypher() throws Exception { String fileName = "graph.cypher"; - String s3Url = getS3Url(fileName); + String s3Url = s3Container.getUrl(fileName); TestUtil.testCall(db, "CALL apoc.graph.fromDB('test',{}) yield graph " + "CALL apoc.export.cypher.graph(graph, $s3,$exportConfig) " + "YIELD nodes, relationships, properties, file, source,format, time " + "RETURN *", map("s3", s3Url, "exportConfig", map("useOptimizations", map("type", "none"), "format", "neo4j-shell")), (r) -> assertResults(s3Url, r, "graph")); - verifyUpload(fileName, EXPECTED_NEO4J_SHELL); + assertStringFileEquals(EXPECTED_NEO4J_SHELL, s3Url); } // -- Separate files tests -- // @Test public void testExportAllCypherNodes() throws Exception { String fileName = "all.cypher"; - String s3Url = getS3Url(fileName); + String s3Url = s3Container.getUrl(fileName); TestUtil.testCall(db, "CALL apoc.export.cypher.all($s3,$exportConfig)", map("s3", s3Url, "exportConfig", exportConfig), (r) -> assertResults(s3Url, r, "database")); - verifyUpload("all.nodes.cypher", EXPECTED_NODES); + getUrlAndAssertEquals(EXPECTED_NODES, "all.nodes.cypher"); } @Test public void testExportAllCypherRelationships() throws Exception { String fileName = "all.cypher"; - String s3Url = getS3Url(fileName); + String s3Url = s3Container.getUrl(fileName); TestUtil.testCall(db, "CALL apoc.export.cypher.all($s3,$exportConfig)", map("s3", s3Url, "exportConfig", exportConfig), (r) -> assertResults(s3Url, r, "database")); - verifyUpload("all.relationships.cypher", EXPECTED_RELATIONSHIPS); + getUrlAndAssertEquals(EXPECTED_RELATIONSHIPS, "all.relationships.cypher"); } @Test public void testExportAllCypherSchema() throws Exception { String fileName = "all.cypher"; - String s3Url = getS3Url(fileName); + String s3Url = s3Container.getUrl(fileName); TestUtil.testCall(db, "CALL apoc.export.cypher.all($s3,$exportConfig)", map("s3", s3Url, "exportConfig", exportConfig), (r) -> assertResults(s3Url, r, "database")); - verifyUpload("all.schema.cypher", EXPECTED_SCHEMA); + getUrlAndAssertEquals(EXPECTED_SCHEMA, "all.schema.cypher"); } @Test public void testExportAllCypherCleanUp() throws Exception { String fileName = "all.cypher"; - String s3Url = getS3Url(fileName); + String s3Url = s3Container.getUrl(fileName); TestUtil.testCall(db, "CALL apoc.export.cypher.all($s3,$exportConfig)", map("s3", s3Url, "exportConfig", exportConfig), (r) -> assertResults(s3Url, r, "database")); - verifyUpload("all.cleanup.cypher", EXPECTED_CLEAN_UP); + getUrlAndAssertEquals(EXPECTED_CLEAN_UP, "all.cleanup.cypher"); } @Test public void testExportGraphCypherNodes() throws Exception { String fileName = "graph.cypher"; - String s3Url = getS3Url(fileName); + String s3Url = s3Container.getUrl(fileName); TestUtil.testCall(db, "CALL apoc.graph.fromDB('test',{}) yield graph " + "CALL apoc.export.cypher.graph(graph, $s3,$exportConfig) " + "YIELD nodes, relationships, properties, file, source,format, time " + "RETURN *", map("s3", s3Url, "exportConfig", exportConfig), (r) -> assertResults(s3Url, r, "graph")); - verifyUpload("graph.nodes.cypher", EXPECTED_NODES); + getUrlAndAssertEquals(EXPECTED_NODES, "graph.nodes.cypher"); } @Test public void testExportGraphCypherRelationships() throws Exception { String fileName = "graph.cypher"; - String s3Url = getS3Url(fileName); + String s3Url = s3Container.getUrl(fileName); TestUtil.testCall(db, "CALL apoc.graph.fromDB('test',{}) yield graph " + "CALL apoc.export.cypher.graph(graph, $s3,$exportConfig) " + "YIELD nodes, relationships, properties, file, source, format, time " + "RETURN *", map("s3", s3Url, "exportConfig", exportConfig), (r) -> assertResults(s3Url, r, "graph")); - verifyUpload("graph.relationships.cypher", EXPECTED_RELATIONSHIPS); + getUrlAndAssertEquals(EXPECTED_RELATIONSHIPS, "graph.relationships.cypher"); } @Test public void testExportGraphCypherSchema() throws Exception { String fileName = "graph.cypher"; - String s3Url = getS3Url(fileName); + String s3Url = s3Container.getUrl(fileName); TestUtil.testCall(db, "CALL apoc.graph.fromDB('test',{}) yield graph " + "CALL apoc.export.cypher.graph(graph, $s3,$exportConfig) " + "YIELD nodes, relationships, properties, file, source,format, time " + "RETURN *", map("s3", s3Url, "exportConfig", exportConfig), (r) -> assertResults(s3Url, r, "graph")); - verifyUpload("graph.schema.cypher", EXPECTED_SCHEMA); + getUrlAndAssertEquals(EXPECTED_SCHEMA, "graph.schema.cypher"); } @Test public void testExportGraphCypherCleanUp() throws Exception { String fileName = "graph.cypher"; - String s3Url = getS3Url(fileName); + String s3Url = s3Container.getUrl(fileName); TestUtil.testCall(db, "CALL apoc.graph.fromDB('test',{}) yield graph " + "CALL apoc.export.cypher.graph(graph, $s3,$exportConfig) " + "YIELD nodes, relationships, properties, file, source,format, time " + "RETURN *", map("s3", s3Url, "exportConfig", exportConfig), (r) -> assertResults(s3Url, r, "graph")); - verifyUpload("graph.cleanup.cypher", EXPECTED_CLEAN_UP); + getUrlAndAssertEquals(EXPECTED_CLEAN_UP, "graph.cleanup.cypher"); } private void assertResults(String fileName, Map r, final String source) { @@ -246,64 +211,64 @@ private void assertResults(String fileName, Map r, final String @Test public void testExportQueryCypherPlainFormat() throws Exception { String fileName = "all.cypher"; - String s3Url = getS3Url(fileName); + String s3Url = s3Container.getUrl(fileName); String query = "MATCH (n) OPTIONAL MATCH p = (n)-[r]-(m) RETURN n,r,m"; TestUtil.testCall(db, "CALL apoc.export.cypher.query($query,$s3,$config)", map("s3", s3Url, "query", query, "config", map("useOptimizations", map("type", "none"), "format", "plain")), (r) -> { }); - verifyUpload(fileName, EXPECTED_PLAIN); + assertStringFileEquals(EXPECTED_PLAIN, s3Url); } @Test public void testExportQueryCypherFormatUpdateAll() throws Exception { String fileName = "all.cypher"; - String s3Url = getS3Url(fileName); + String s3Url = s3Container.getUrl(fileName); String query = "MATCH (n) OPTIONAL MATCH p = (n)-[r]-(m) RETURN n,r,m"; TestUtil.testCall(db, "CALL apoc.export.cypher.query($query,$s3,$config)", map("s3", s3Url, "query", query, "config", map("useOptimizations", map("type", "none"), "format", "neo4j-shell", "cypherFormat", "updateAll")), (r) -> { }); - verifyUpload(fileName, EXPECTED_NEO4J_MERGE); + assertStringFileEquals(EXPECTED_NEO4J_MERGE, s3Url); } @Test public void testExportQueryCypherFormatAddStructure() throws Exception { String fileName = "all.cypher"; - String s3Url = getS3Url(fileName); + String s3Url = s3Container.getUrl(fileName); String query = "MATCH (n) OPTIONAL MATCH p = (n)-[r]-(m) RETURN n,r,m"; TestUtil.testCall(db, "CALL apoc.export.cypher.query($query,$s3,$config)", map("s3", s3Url, "query", query, "config", map("useOptimizations", map("type", "none"), "format", "neo4j-shell", "cypherFormat", "addStructure")), (r) -> { }); - verifyUpload(fileName, EXPECTED_NODES_MERGE_ON_CREATE_SET + EXPECTED_SCHEMA_EMPTY + EXPECTED_RELATIONSHIPS + EXPECTED_CLEAN_UP_EMPTY); + assertStringFileEquals(EXPECTED_NODES_MERGE_ON_CREATE_SET + EXPECTED_SCHEMA_EMPTY + EXPECTED_RELATIONSHIPS + EXPECTED_CLEAN_UP_EMPTY, s3Url); } @Test public void testExportQueryCypherFormatUpdateStructure() throws Exception { String fileName = "all.cypher"; - String s3Url = getS3Url(fileName); + String s3Url = s3Container.getUrl(fileName); String query = "MATCH (n) OPTIONAL MATCH p = (n)-[r]-(m) RETURN n,r,m"; TestUtil.testCall(db, "CALL apoc.export.cypher.query($query,$s3,$config)", map("s3", s3Url, "query", query, "config", map("useOptimizations", map("type", "none"), "format", "neo4j-shell", "cypherFormat", "updateStructure")), (r) -> { }); - verifyUpload(fileName, EXPECTED_NODES_EMPTY + EXPECTED_SCHEMA_EMPTY + EXPECTED_RELATIONSHIPS_MERGE_ON_CREATE_SET + EXPECTED_CLEAN_UP_EMPTY); + assertStringFileEquals(EXPECTED_NODES_EMPTY + EXPECTED_SCHEMA_EMPTY + EXPECTED_RELATIONSHIPS_MERGE_ON_CREATE_SET + EXPECTED_CLEAN_UP_EMPTY, s3Url); } @Test public void testExportSchemaCypher() throws Exception { String fileName = "onlySchema.cypher"; - String s3Url = getS3Url(fileName); + String s3Url = s3Container.getUrl(fileName); TestUtil.testCall(db, "CALL apoc.export.cypher.schema($s3,$exportConfig)", map("s3", s3Url, "exportConfig", exportConfig), (r) -> { }); - verifyUpload(fileName, EXPECTED_ONLY_SCHEMA_NEO4J_SHELL); + assertStringFileEquals(EXPECTED_ONLY_SCHEMA_NEO4J_SHELL, s3Url); } @Test public void testExportSchemaCypherShell() throws Exception { String fileName = "onlySchema.cypher"; - String s3Url = getS3Url(fileName); + String s3Url = s3Container.getUrl(fileName); TestUtil.testCall(db, "CALL apoc.export.cypher.schema($s3,$exportConfig)", map("s3", s3Url, "exportConfig", map("useOptimizations", map("type", "none"), "format", "cypher-shell")), (r) -> {}); - verifyUpload(fileName, EXPECTED_ONLY_SCHEMA_CYPHER_SHELL); + assertStringFileEquals(EXPECTED_ONLY_SCHEMA_CYPHER_SHELL, s3Url); } @Test @@ -314,12 +279,12 @@ public void testExportCypherNodePoint() throws IOException { "-[:FRIEND_OF {place2d:point({ longitude: 56.7, latitude: 12.78 })}]->" + "(:Bar {place3d:point({ longitude: 12.78, latitude: 56.7, height: 100 })})"); String fileName = "temporalPoint.cypher"; - String s3Url = getS3Url(fileName); + String s3Url = s3Container.getUrl(fileName); String query = "MATCH (n:Test)-[r]-(m) RETURN n,r,m"; TestUtil.testCall(db, "CALL apoc.export.cypher.query($query,$s3,$config)", map("s3", s3Url, "query", query, "config", map("useOptimizations", map("type", "none"),"format", "neo4j-shell")), (r) -> {}); - verifyUpload(fileName, EXPECTED_CYPHER_POINT); + assertStringFileEquals(EXPECTED_CYPHER_POINT, s3Url); } @Test @@ -331,12 +296,12 @@ public void testExportCypherNodeDate() throws IOException { "-[:FRIEND_OF {date:date('2018-10-30')}]->" + "(:Bar {datetime:datetime('2018-10-30T12:50:35.556')})"); String fileName = "temporalDate.cypher"; - String s3Url = getS3Url(fileName); + String s3Url = s3Container.getUrl(fileName); String query = "MATCH (n:Test)-[r]-(m) RETURN n,r,m"; TestUtil.testCall(db, "CALL apoc.export.cypher.query($query,$s3,$config)", map("s3", s3Url, "query", query, "config", map("useOptimizations", map("type", "none"),"format", "neo4j-shell")), (r) -> {}); - verifyUpload(fileName, EXPECTED_CYPHER_DATE); + assertStringFileEquals(EXPECTED_CYPHER_DATE, s3Url); } @Test @@ -347,12 +312,12 @@ public void testExportCypherNodeTime() throws IOException { "-[:FRIEND_OF {t:time('125035.556+0100')}]->" + "(:Bar {datetime:datetime('2018-10-30T12:50:35.556+0100')})"); String fileName = "temporalTime.cypher"; - String s3Url = getS3Url(fileName); + String s3Url = s3Container.getUrl(fileName); String query = "MATCH (n:Test)-[r]-(m) RETURN n,r,m"; TestUtil.testCall(db, "CALL apoc.export.cypher.query($query,$s3,$config)", map("s3", s3Url, "query", query, "config", map("useOptimizations", map("type", "none"),"format", "neo4j-shell")), (r) -> {}); - verifyUpload(fileName, EXPECTED_CYPHER_TIME); + assertStringFileEquals(EXPECTED_CYPHER_TIME, s3Url); } @Test @@ -362,100 +327,100 @@ public void testExportCypherNodeDuration() throws IOException { "-[:FRIEND_OF {duration:duration('P5M1.5D')}]->" + "(:Bar {duration:duration('P5M1.5D')})"); String fileName = "temporalDuration.cypher"; - String s3Url = getS3Url(fileName); + String s3Url = s3Container.getUrl(fileName); String query = "MATCH (n:Test)-[r]-(m) RETURN n,r,m"; TestUtil.testCall(db, "CALL apoc.export.cypher.query($query,$s3,$config)", map("s3", s3Url, "query", query, "config", map("useOptimizations", map("type", "none"),"format", "neo4j-shell")), (r) -> {}); - verifyUpload(fileName, EXPECTED_CYPHER_DURATION); + assertStringFileEquals(EXPECTED_CYPHER_DURATION, s3Url); } @Test public void testExportWithAscendingLabels() throws IOException { db.executeTransactionally("CREATE (f:User:User1:User0:User12 {name:'Alan'})"); String fileName = "ascendingLabels.cypher"; - String s3Url = getS3Url(fileName); + String s3Url = s3Container.getUrl(fileName); String query = "MATCH (f:User) WHERE f.name='Alan' RETURN f"; TestUtil.testCall(db, "CALL apoc.export.cypher.query($query,$s3,$config)", map("s3", s3Url, "query", query, "config", map("useOptimizations", map("type", "none"),"format", "neo4j-shell")), (r) -> {}); - verifyUpload(fileName, EXPECTED_CYPHER_LABELS_ASCENDEND); + assertStringFileEquals(EXPECTED_CYPHER_LABELS_ASCENDEND, s3Url); } @Test public void testExportAllCypherDefaultWithUnwindBatchSizeOptimized() throws Exception { String fileName = "allDefaultOptimized.cypher"; - String s3Url = getS3Url(fileName); + String s3Url = s3Container.getUrl(fileName); TestUtil.testCall(db, "CALL apoc.export.cypher.all($s3,{useOptimizations: { type: 'unwind_batch', unwindBatchSize: 2}, format: 'neo4j-shell'})", map("s3", s3Url), (r) -> assertResultsOptimized(s3Url, r)); - verifyUpload(fileName, EXPECTED_NEO4J_OPTIMIZED_BATCH_SIZE); + assertStringFileEquals(EXPECTED_NEO4J_OPTIMIZED_BATCH_SIZE, s3Url); } @Test public void testExportAllCypherDefaultOptimized() throws Exception { String fileName = "allDefaultOptimized.cypher"; - String s3Url = getS3Url(fileName); + String s3Url = s3Container.getUrl(fileName); TestUtil.testCall(db, "CALL apoc.export.cypher.all($s3, $exportConfig)", map("s3", s3Url, "exportConfig", map("format", "neo4j-shell")), (r) -> assertResultsOptimized(s3Url, r)); - verifyUpload(fileName, EXPECTED_NEO4J_OPTIMIZED); + assertStringFileEquals(EXPECTED_NEO4J_OPTIMIZED, s3Url); } @Test public void testExportAllCypherDefaultSeparatedFilesOptimized() throws Exception { String fileName = "allDefaultOptimized.cypher"; - String s3Url = getS3Url(fileName); + String s3Url = s3Container.getUrl(fileName); TestUtil.testCall(db, "CALL apoc.export.cypher.all($s3, $exportConfig)", map("s3", s3Url, "exportConfig", map("separateFiles", true, "format", "neo4j-shell")), (r) -> assertResultsOptimized(s3Url, r)); - verifyUpload("allDefaultOptimized.nodes.cypher", EXPECTED_NODES_OPTIMIZED); - verifyUpload("allDefaultOptimized.relationships.cypher", EXPECTED_RELATIONSHIPS_OPTIMIZED); - verifyUpload("allDefaultOptimized.schema.cypher", EXPECTED_SCHEMA); - verifyUpload("allDefaultOptimized.cleanup.cypher", EXPECTED_CLEAN_UP); + getUrlAndAssertEquals(EXPECTED_NODES_OPTIMIZED, "allDefaultOptimized.nodes.cypher"); + getUrlAndAssertEquals(EXPECTED_RELATIONSHIPS_OPTIMIZED, "allDefaultOptimized.relationships.cypher"); + getUrlAndAssertEquals(EXPECTED_SCHEMA, "allDefaultOptimized.schema.cypher"); + getUrlAndAssertEquals(EXPECTED_CLEAN_UP, "allDefaultOptimized.cleanup.cypher"); } @Test public void testExportAllCypherCypherShellWithUnwindBatchSizeOptimized() throws Exception { String fileName = "allCypherShellOptimized.cypher"; - String s3Url = getS3Url(fileName); + String s3Url = s3Container.getUrl(fileName); TestUtil.testCall(db, "CALL apoc.export.cypher.all($s3,{format:'cypher-shell', useOptimizations: {type: 'unwind_batch'}})", map("s3", s3Url), (r) -> assertResultsOptimized(s3Url, r)); - verifyUpload(fileName, EXPECTED_CYPHER_SHELL_OPTIMIZED_BATCH_SIZE); + assertStringFileEquals(EXPECTED_CYPHER_SHELL_OPTIMIZED_BATCH_SIZE, s3Url); } @Test public void testExportAllCypherCypherShellOptimized() throws Exception { String fileName = "allCypherShellOptimized.cypher"; - String s3Url = getS3Url(fileName); + String s3Url = s3Container.getUrl(fileName); TestUtil.testCall(db, "CALL apoc.export.cypher.all($s3,{format:'cypher-shell'})", map("s3", s3Url), (r) -> assertResultsOptimized(s3Url, r)); - verifyUpload(fileName, EXPECTED_CYPHER_SHELL_OPTIMIZED); + assertStringFileEquals(EXPECTED_CYPHER_SHELL_OPTIMIZED, s3Url); } @Test public void testExportAllCypherPlainWithUnwindBatchSizeOptimized() throws Exception { String fileName = "allPlainOptimized.cypher"; - String s3Url = getS3Url(fileName); + String s3Url = s3Container.getUrl(fileName); TestUtil.testCall(db, "CALL apoc.export.cypher.all($s3,{format:'plain', useOptimizations: { type: 'unwind_batch', unwindBatchSize: 2}})", map("s3", s3Url), (r) -> assertResultsOptimized(s3Url, r)); - verifyUpload(fileName, EXPECTED_PLAIN_OPTIMIZED_BATCH_SIZE); + assertStringFileEquals(EXPECTED_PLAIN_OPTIMIZED_BATCH_SIZE, s3Url); } @Test public void testExportAllCypherPlainAddStructureWithUnwindBatchSizeOptimized() throws Exception { String fileName = "allPlainAddStructureOptimized.cypher"; - String s3Url = getS3Url(fileName); + String s3Url = s3Container.getUrl(fileName); TestUtil.testCall(db, "CALL apoc.export.cypher.all($s3,{format:'plain', cypherFormat: 'addStructure', useOptimizations: { type: 'unwind_batch', unwindBatchSize: 2}})", map("s3", s3Url), (r) -> assertResultsOptimized(s3Url, r)); - verifyUpload(fileName, EXPECTED_PLAIN_ADD_STRUCTURE_UNWIND); + assertStringFileEquals(EXPECTED_PLAIN_ADD_STRUCTURE_UNWIND, s3Url); } @Test public void testExportAllCypherPlainUpdateStructureWithUnwindBatchSizeOptimized() throws Exception { String fileName = "allPlainUpdateStructureOptimized.cypher"; - String s3Url = getS3Url(fileName); + String s3Url = s3Container.getUrl(fileName); TestUtil.testCall(db, "CALL apoc.export.cypher.all($s3,{format:'plain', cypherFormat: 'updateStructure', useOptimizations: { type: 'unwind_batch', unwindBatchSize: 2}})", map("s3", s3Url), (r) -> { assertEquals(0L, r.get("nodes")); @@ -465,52 +430,52 @@ public void testExportAllCypherPlainUpdateStructureWithUnwindBatchSizeOptimized( assertEquals("cypher", r.get("format")); assertTrue("Should get time greater than 0",((long) r.get("time")) >= 0); }); - verifyUpload(fileName, EXPECTED_PLAIN_UPDATE_STRUCTURE_UNWIND); + assertStringFileEquals(EXPECTED_PLAIN_UPDATE_STRUCTURE_UNWIND, s3Url); } @Test public void testExportAllCypherPlainUpdateAllWithUnwindBatchSizeOptimized() throws Exception { String fileName = "allPlainUpdateAllOptimized.cypher"; - String s3Url = getS3Url(fileName); + String s3Url = s3Container.getUrl(fileName); TestUtil.testCall(db, "CALL apoc.export.cypher.all($s3,{format:'plain', cypherFormat: 'updateAll', useOptimizations: { type: 'unwind_batch', unwindBatchSize: 2}})", map("s3", s3Url), (r) -> assertResultsOptimized(s3Url, r)); - verifyUpload(fileName, EXPECTED_UPDATE_ALL_UNWIND); + assertStringFileEquals(EXPECTED_UPDATE_ALL_UNWIND, s3Url); } @Test public void testExportQueryCypherShellWithUnwindBatchSizeWithBatchSizeOptimized() throws Exception { String fileName = "allPlainOptimized.cypher"; - String s3Url = getS3Url(fileName); + String s3Url = s3Container.getUrl(fileName); TestUtil.testCall(db, "CALL apoc.export.cypher.all($s3,{format:'cypher-shell', useOptimizations: { type: 'unwind_batch', unwindBatchSize: 2}, batchSize: 2})", map("s3", s3Url), (r) -> assertResultsOptimized(s3Url, r)); - verifyUpload(fileName, EXPECTED_QUERY_CYPHER_SHELL_OPTIMIZED_UNWIND); + assertStringFileEquals(EXPECTED_QUERY_CYPHER_SHELL_OPTIMIZED_UNWIND, s3Url); } @Test public void testExportQueryCypherShellWithUnwindBatchSizeWithBatchSizeOddDataset() throws Exception { String fileName = "allPlainOdd.cypher"; - String s3Url = getS3Url(fileName); + String s3Url = s3Container.getUrl(fileName); TestUtil.testCall(db, "CALL apoc.export.cypher.all($s3,{format:'cypher-shell', useOptimizations: { type: 'unwind_batch', unwindBatchSize: 2}, batchSize: 2})", map("s3", s3Url), (r) -> assertResultsOdd(s3Url, r)); - verifyUpload(fileName, EXPECTED_QUERY_CYPHER_SHELL_OPTIMIZED_ODD); + assertStringFileEquals(EXPECTED_QUERY_CYPHER_SHELL_OPTIMIZED_ODD, s3Url); } @Test public void testExportQueryCypherShellUnwindBatchParamsWithOddDataset() throws Exception { String fileName = "allPlainOdd.cypher"; - String s3Url = getS3Url(fileName); + String s3Url = s3Container.getUrl(fileName); TestUtil.testCall(db, "CALL apoc.export.cypher.all($s3,{format:'cypher-shell', useOptimizations: { type: 'unwind_batch_params', unwindBatchSize: 2}, batchSize:2})", map("s3", s3Url), (r) -> assertResultsOdd(s3Url, r)); - verifyUpload(fileName, EXPECTED_QUERY_CYPHER_SHELL_PARAMS_OPTIMIZED_ODD); + assertStringFileEquals(EXPECTED_QUERY_CYPHER_SHELL_PARAMS_OPTIMIZED_ODD, s3Url); } @Test public void testExportQueryCypherShellUnwindBatchParamsWithOddBatchSizeOddDataset() throws Exception { db.executeTransactionally("CREATE (:Bar {name:'bar3',age:35}), (:Bar {name:'bar4',age:36})"); String fileName = "allPlainOddNew.cypher"; - String s3Url = getS3Url(fileName); + String s3Url = s3Container.getUrl(fileName); TestUtil.testCall(db, "CALL apoc.export.cypher.all($s3,{format:'cypher-shell', useOptimizations: { type: 'unwind_batch_params', unwindBatchSize: 2}, batchSize:3})", map("s3", s3Url), (r) -> {}); @@ -554,7 +519,12 @@ public void testExportQueryCypherShellUnwindBatchParamsWithOddBatchSizeOddDatase .replace(NEO4J_SHELL.commit(), CYPHER_SHELL.commit()) .replace(NEO4J_SHELL.schemaAwait(), EXPECTED_INDEXES_AWAIT) .replace(NEO4J_SHELL.schemaAwait(), CYPHER_SHELL.schemaAwait()); - verifyUpload(fileName, expected); + assertStringFileEquals(expected, s3Url); + } + + private void getUrlAndAssertEquals(String expected, String fileName) { + final String urlFile = s3Container.getUrl(fileName); + S3TestUtil.assertStringFileEquals(expected, urlFile); } private void assertResultsOptimized(String fileName, Map r) { diff --git a/core/src/test/java/apoc/export/graphml/ExportGraphMLS3Test.java b/core/src/test/java/apoc/export/graphml/ExportGraphMLS3Test.java index 8df0f5b0d..92ca4a098 100644 --- a/core/src/test/java/apoc/export/graphml/ExportGraphMLS3Test.java +++ b/core/src/test/java/apoc/export/graphml/ExportGraphMLS3Test.java @@ -1,225 +1,82 @@ package apoc.export.graphml; -import apoc.graph.Graphs; import apoc.util.TestUtil; -import apoc.util.s3.S3TestUtil; +import apoc.util.s3.S3BaseTest; import org.junit.Before; import org.junit.Rule; import org.junit.Test; -import org.junit.Ignore; import org.junit.rules.TestName; -import org.neo4j.configuration.GraphDatabaseSettings; import org.neo4j.test.rule.DbmsRule; import org.neo4j.test.rule.ImpermanentDbmsRule; -import org.xmlunit.builder.DiffBuilder; -import org.xmlunit.diff.DefaultNodeMatcher; -import org.xmlunit.diff.Diff; -import org.xmlunit.diff.ElementSelector; -import org.xmlunit.util.Nodes; -import javax.xml.namespace.QName; -import java.io.File; import java.io.FileNotFoundException; -import java.io.IOException; -import java.nio.file.Paths; -import java.util.Arrays; -import java.util.List; import java.util.Map; -import java.util.Optional; -import static apoc.ApocConfig.*; +import static apoc.export.graphml.ExportGraphMLTestUtil.EXPECTED_FALSE; +import static apoc.export.graphml.ExportGraphMLTestUtil.EXPECTED_TYPES; +import static apoc.export.graphml.ExportGraphMLTestUtil.EXPECTED_TYPES_PATH; +import static apoc.export.graphml.ExportGraphMLTestUtil.EXPECTED_TYPES_PATH_CAMEL_CASE; +import static apoc.export.graphml.ExportGraphMLTestUtil.EXPECTED_TYPES_PATH_CAPTION; +import static apoc.export.graphml.ExportGraphMLTestUtil.EXPECTED_TYPES_PATH_WRONG_CAPTION; +import static apoc.export.graphml.ExportGraphMLTestUtil.assertXMLEquals; +import static apoc.export.graphml.ExportGraphMLTestUtil.setUpGraphMl; import static apoc.util.MapUtil.map; +import static apoc.util.s3.S3TestUtil.readS3FileToString; import static org.junit.Assert.*; -import static org.xmlunit.diff.ElementSelectors.byName; -@Ignore("To use this test, you need to set the S3 bucket and region to a valid endpoint " + - "and have your access key and secret key setup in your environment.") -public class ExportGraphMLS3Test { - private static String S3_BUCKET_NAME = null; - - public static final List ATTRIBUTES_CONTAINING_NODE_IDS = Arrays.asList("id", "source", "target"); - - public static final String GRAPH = "%n"; - public static final String HEADER = "%n" + - "%n"; - public static final String KEY_TYPES_FALSE = "%n" + - "%n" + - "%n" + - "%n"+ - "%n" + - "%n" + - "%n"; - public static final String KEY_TYPES = "%n" + - "%n" + - "%n" + - "%n"+ - "%n" + - "%n" + - "%n"; - public static final String KEY_TYPES_PATH = "%n" + - "%n" + - "%n"+ - "%n" + - "%n" + - "%n" + - "%n" + - "%n"; - public static final String KEY_TYPES_CAMEL_CASE = "%n" + - "%n" + - "%n" + - "%n" + - "%n" + - "%n" + - "%n"; - public static final String DATA = ":Foo:Foo0:Foo2{\"crs\":\"wgs-84-3d\",\"latitude\":56.7,\"longitude\":12.78,\"height\":100.0}foo2018-10-10%n" + - ":Bar42bar{\"crs\":\"wgs-84\",\"latitude\":56.7,\"longitude\":12.78,\"height\":null}%n" + - ":Bar12[1,2,3]%n" + - "KNOWS%n"; - public static final String DATA_CAMEL_CASE = - ":Foo:Foo0:Foo2foofoo%n" + - ":Barbarbar42%n" + - "KNOWSKNOWS%n"; - - public static final String FOOTER = "%n" + - ""; - - public static final String DATA_PATH = ":Foo:Foo0:Foo2foo{\"crs\":\"wgs-84-3d\",\"latitude\":56.7,\"longitude\":12.78,\"height\":100.0}foo2018-10-10%n" + - ":Barbar42bar{\"crs\":\"wgs-84\",\"latitude\":56.7,\"longitude\":12.78,\"height\":null}%n" + - "KNOWSKNOWS%n"; - - public static final String DATA_PATH_CAPTION = ":Foo:Foo0:Foo2foo{\"crs\":\"wgs-84-3d\",\"latitude\":56.7,\"longitude\":12.78,\"height\":100.0}foo2018-10-10%n" + - ":Barbar42bar{\"crs\":\"wgs-84\",\"latitude\":56.7,\"longitude\":12.78,\"height\":null}%n" + - "KNOWSKNOWS%n"; - - public static final String DATA_PATH_CAPTION_DEFAULT = ":Foo:Foo0:Foo2point({x: 56.7, y: 12.78, z: 100.0, crs: 'wgs-84-3d'}){\"crs\":\"wgs-84-3d\",\"latitude\":56.7,\"longitude\":12.78,\"height\":100.0}foo2018-10-10%n" + - ":Bar4242bar{\"crs\":\"wgs-84\",\"latitude\":56.7,\"longitude\":12.78,\"height\":null}%n" + - "KNOWSKNOWS%n"; - - private static final String EXPECTED_TYPES_PATH = String.format(HEADER + KEY_TYPES_PATH + GRAPH + DATA_PATH + FOOTER); - private static final String EXPECTED_TYPES_PATH_CAPTION = String.format(HEADER + KEY_TYPES_PATH + GRAPH + DATA_PATH_CAPTION + FOOTER); - private static final String EXPECTED_TYPES_PATH_WRONG_CAPTION = String.format(HEADER + KEY_TYPES_PATH + GRAPH + DATA_PATH_CAPTION_DEFAULT + FOOTER); - private static final String EXPECTED_TYPES = String.format(HEADER + KEY_TYPES + GRAPH + DATA + FOOTER); - private static final String EXPECTED_FALSE = String.format(HEADER + KEY_TYPES_FALSE + GRAPH + DATA + FOOTER); - private static final String EXPECTED_TYPES_PATH_CAMEL_CASE = String.format(HEADER + KEY_TYPES_CAMEL_CASE + GRAPH + DATA_CAMEL_CASE + FOOTER); +public class ExportGraphMLS3Test extends S3BaseTest { @Rule public TestName testName = new TestName(); - private static final String TEST_WITH_NO_IMPORT = "WithNoImportConfig"; - private static final String TEST_WITH_NO_EXPORT = "WithNoExportConfig"; - @Rule - public DbmsRule db = new ImpermanentDbmsRule() - .withSetting(GraphDatabaseSettings.load_csv_file_url_root, directory.toPath().toAbsolutePath()); - - private static File directory = new File("target/import"); - - static { //noinspection ResultOfMethodCallIgnored - directory.mkdirs(); - } - - private static String getS3Url(String key) { - return String.format("s3://:@/%s/%s", S3_BUCKET_NAME, key); - } - - private static String readFile(String fileName) { - return TestUtil.readFileToString(new File(directory, fileName)); - } - - private void verifyUpload(String s3Url, String fileName, String expected) throws IOException { - S3TestUtil.readFile(s3Url, Paths.get(directory.toString(), fileName).toString()); - assertXMLEquals(expected, readFile(fileName)); - } - - private static String getEnvVar(String envVarKey) throws Exception { - return Optional.ofNullable(System.getenv(envVarKey)).orElseThrow( - () -> new Exception(String.format("%s is not set in the environment", envVarKey)) - ); - } + public DbmsRule db = new ImpermanentDbmsRule(); @Before public void setUp() throws Exception { - if (S3_BUCKET_NAME == null) { - S3_BUCKET_NAME = getEnvVar("S3_BUCKET_NAME"); - } - - TestUtil.registerProcedure(db, ExportGraphML.class, Graphs.class); - - apocConfig().setProperty(APOC_EXPORT_FILE_ENABLED, Boolean.toString(!testName.getMethodName().endsWith(TEST_WITH_NO_EXPORT))); - apocConfig().setProperty(APOC_IMPORT_FILE_ENABLED, Boolean.toString(!testName.getMethodName().endsWith(TEST_WITH_NO_IMPORT))); - apocConfig().setProperty(APOC_IMPORT_FILE_USE_NEO4J_CONFIG, false); - - db.executeTransactionally("CREATE (f:Foo:Foo2:Foo0 {name:'foo', born:Date('2018-10-10'), place:point({ longitude: 56.7, latitude: 12.78, height: 100 })})-[:KNOWS]->(b:Bar {name:'bar',age:42, place:point({ longitude: 56.7, latitude: 12.78})}),(c:Bar {age:12,values:[1,2,3]})"); + setUpGraphMl(db, testName); } @Test public void testExportAllGraphML() throws Exception { String fileName = "all.graphml"; - String s3Url = getS3Url(fileName); + String s3Url = s3Container.getUrl(fileName); TestUtil.testCall(db, "CALL apoc.export.graphml.all($s3, null)", map("s3", s3Url), (r) -> assertResults(s3Url, r, "database")); - verifyUpload(s3Url, fileName, EXPECTED_FALSE); + assertXmlFileEquals(EXPECTED_FALSE, s3Url); } @Test public void testExportGraphGraphML() throws Exception { String fileName = "graph.graphml"; - String s3Url = getS3Url(fileName); + String s3Url = s3Container.getUrl(fileName); TestUtil.testCall(db, "CALL apoc.graph.fromDB('test',{}) yield graph " + "CALL apoc.export.graphml.graph(graph, $s3, null) " + "YIELD nodes, relationships, properties, file, source, format, time " + "RETURN *", map("s3", s3Url), (r) -> assertResults(s3Url, r, "graph")); - verifyUpload(s3Url, fileName, EXPECTED_FALSE); - } - - private void assertXMLEquals(Object output, String xmlString) { - Diff myDiff = DiffBuilder.compare(xmlString) - .withTest(output) - .checkForSimilar() - .ignoreWhitespace() - .withAttributeFilter(attr -> !ATTRIBUTES_CONTAINING_NODE_IDS.contains(attr.getLocalName())) // ignore id properties - .withNodeMatcher(new DefaultNodeMatcher((ElementSelector) (controlElement, testElement) -> { - if (!byName.canBeCompared(controlElement, testElement)) { - return false; - } - Map cAttrs = Nodes.getAttributes(controlElement); - Map tAttrs = Nodes.getAttributes(testElement); - if (cAttrs.size() != tAttrs.size()) { - return false; - } - for (Map.Entry e: cAttrs.entrySet()) { - if ((!ATTRIBUTES_CONTAINING_NODE_IDS.contains(e.getKey().getLocalPart())) - && (!e.getValue().equals(tAttrs.get(e.getKey())))) { - return false; - } - } - return true; - })) - .build(); - - assertFalse(myDiff.toString(), myDiff.hasDifferences()); + assertXmlFileEquals(EXPECTED_FALSE, s3Url); } @Test public void testExportGraphGraphMLTypes() throws Exception { String fileName = "graph.graphml"; - String s3Url = getS3Url(fileName); + String s3Url = s3Container.getUrl(fileName); TestUtil.testCall(db, "CALL apoc.graph.fromDB('test',{}) yield graph " + "CALL apoc.export.graphml.graph(graph, $s3,{useTypes:true}) " + "YIELD nodes, relationships, properties, file, source,format, time " + "RETURN *", map("s3", s3Url), (r) -> assertResults(s3Url, r, "graph")); - verifyUpload(s3Url, fileName, EXPECTED_TYPES); + assertXmlFileEquals(EXPECTED_TYPES, s3Url); } @Test public void testExportGraphGraphMLQueryGephi() throws Exception { String fileName = "query.graphml"; - String s3Url = getS3Url(fileName); + String s3Url = s3Container.getUrl(fileName); TestUtil.testCall(db, "call apoc.export.graphml.query('MATCH p=()-[r]->() RETURN p limit 1000',$s3,{useTypes:true, format: 'gephi'}) ", map("s3", s3Url), (r) -> { @@ -234,13 +91,13 @@ public void testExportGraphGraphMLQueryGephi() throws Exception { assertEquals("graphml", r.get("format")); assertTrue("Should get time greater than 0",((long) r.get("time")) > 0); }); - verifyUpload(s3Url, fileName, EXPECTED_TYPES_PATH); + assertXmlFileEquals(EXPECTED_TYPES_PATH, s3Url); } @Test public void testExportGraphGraphMLQueryGephiWithArrayCaption() throws Exception { String fileName = "query.graphml"; - String s3Url = getS3Url(fileName); + String s3Url = s3Container.getUrl(fileName); TestUtil.testCall(db, "call apoc.export.graphml.query('MATCH p=()-[r]->() RETURN p limit 1000',$s3,{useTypes:true, format: 'gephi', caption: ['bar','name','foo']}) ", map("s3", s3Url), (r) -> { @@ -255,14 +112,14 @@ public void testExportGraphGraphMLQueryGephiWithArrayCaption() throws Exception assertEquals("graphml", r.get("format")); assertTrue("Should get time greater than 0",((long) r.get("time")) > 0); }); - verifyUpload(s3Url, fileName, EXPECTED_TYPES_PATH_CAPTION); + assertXmlFileEquals(EXPECTED_TYPES_PATH_CAPTION, s3Url); } @Test public void testExportGraphGraphMLQueryGephiWithArrayCaptionWrong() throws Exception { String fileName = "query.graphml"; - String s3Url = getS3Url(fileName); - TestUtil.testCall(db, "call apoc.export.graphml.query('MATCH p=()-[r]->() RETURN p limit 1000',$s3,{useTypes:true, format: 'gephi', caption: ['a','b','c']}) ", + String s3Url = s3Container.getUrl(fileName); + TestUtil.testCall(db, "call apoc.export.graphml.query('MATCH p=()-[r]->() RETURN p limit 1000',$s3,{useTypes:true, format: 'gephi', caption: ['c','d','e']}) ", map("s3", s3Url), (r) -> { assertEquals(2L, r.get("nodes")); @@ -276,7 +133,7 @@ public void testExportGraphGraphMLQueryGephiWithArrayCaptionWrong() throws Excep assertEquals("graphml", r.get("format")); assertTrue("Should get time greater than 0",((long) r.get("time")) > 0); }); - verifyUpload(s3Url, fileName, EXPECTED_TYPES_PATH_WRONG_CAPTION); + assertXmlFileEquals(EXPECTED_TYPES_PATH_WRONG_CAPTION, s3Url); } @Test @@ -284,7 +141,7 @@ public void testExportGraphmlQueryWithStringCaptionCamelCase() throws FileNotFou db.executeTransactionally("MATCH (n) detach delete (n)"); db.executeTransactionally("CREATE (f:Foo:Foo2:Foo0 {firstName:'foo'})-[:KNOWS]->(b:Bar {name:'bar',ageNow:42}),(c:Bar {age:12,values:[1,2,3]})"); String fileName = "query.graphml"; - String s3Url = getS3Url(fileName); + String s3Url = s3Container.getUrl(fileName); TestUtil.testCall(db, "call apoc.export.graphml.query('MATCH p=()-[r]->() RETURN p limit 1000',$s3,{useTypes:true, format: 'gephi'}) ", map("s3", s3Url), (r) -> { @@ -299,7 +156,12 @@ public void testExportGraphmlQueryWithStringCaptionCamelCase() throws FileNotFou assertEquals("graphml", r.get("format")); assertTrue("Should get time greater than 0",((long) r.get("time")) > 0); }); - verifyUpload(s3Url, fileName, EXPECTED_TYPES_PATH_CAMEL_CASE); + assertXmlFileEquals(EXPECTED_TYPES_PATH_CAMEL_CASE, s3Url); + } + + private void assertXmlFileEquals(String expected, String s3Url) { + final String actual = readS3FileToString(s3Url); + assertXMLEquals(expected, actual); } private void assertResults(String fileName, Map r, final String source) { diff --git a/core/src/test/java/apoc/export/graphml/ExportGraphMLTest.java b/core/src/test/java/apoc/export/graphml/ExportGraphMLTest.java index 4bcf259ca..be18a4b75 100644 --- a/core/src/test/java/apoc/export/graphml/ExportGraphMLTest.java +++ b/core/src/test/java/apoc/export/graphml/ExportGraphMLTest.java @@ -1,6 +1,5 @@ package apoc.export.graphml; -import apoc.graph.Graphs; import apoc.util.BinaryTestUtil; import apoc.util.CompressionAlgo; import apoc.util.CompressionConfig; @@ -21,13 +20,7 @@ import org.neo4j.graphdb.ResourceIterator; import org.neo4j.test.rule.DbmsRule; import org.neo4j.test.rule.ImpermanentDbmsRule; -import org.xmlunit.builder.DiffBuilder; -import org.xmlunit.diff.DefaultNodeMatcher; -import org.xmlunit.diff.Diff; -import org.xmlunit.diff.ElementSelector; -import org.xmlunit.util.Nodes; -import javax.xml.namespace.QName; import java.io.File; import java.io.FileWriter; import java.nio.charset.StandardCharsets; @@ -36,11 +29,22 @@ import java.util.Map; import java.util.Set; -import static apoc.ApocConfig.APOC_EXPORT_FILE_ENABLED; -import static apoc.ApocConfig.APOC_IMPORT_FILE_ENABLED; -import static apoc.ApocConfig.APOC_IMPORT_FILE_USE_NEO4J_CONFIG; import static apoc.ApocConfig.EXPORT_TO_FILE_ERROR; -import static apoc.ApocConfig.apocConfig; +import static apoc.export.graphml.ExportGraphMLTestUtil.EXPECTED_DATA; +import static apoc.export.graphml.ExportGraphMLTestUtil.EXPECTED_FALSE; +import static apoc.export.graphml.ExportGraphMLTestUtil.EXPECTED_READ_NODE_EDGE; +import static apoc.export.graphml.ExportGraphMLTestUtil.EXPECTED_TINKER; +import static apoc.export.graphml.ExportGraphMLTestUtil.EXPECTED_TYPES; +import static apoc.export.graphml.ExportGraphMLTestUtil.EXPECTED_TYPES_EMPTY; +import static apoc.export.graphml.ExportGraphMLTestUtil.EXPECTED_TYPES_NO_DATA_KEY; +import static apoc.export.graphml.ExportGraphMLTestUtil.EXPECTED_TYPES_PATH; +import static apoc.export.graphml.ExportGraphMLTestUtil.EXPECTED_TYPES_PATH_CAMEL_CASE; +import static apoc.export.graphml.ExportGraphMLTestUtil.EXPECTED_TYPES_PATH_CAPTION; +import static apoc.export.graphml.ExportGraphMLTestUtil.EXPECTED_TYPES_PATH_CAPTION_TINKER; +import static apoc.export.graphml.ExportGraphMLTestUtil.EXPECTED_TYPES_PATH_WRONG_CAPTION; +import static apoc.export.graphml.ExportGraphMLTestUtil.EXPECTED_TYPES_WITHOUT_CHAR_DATA_KEYS; +import static apoc.export.graphml.ExportGraphMLTestUtil.assertXMLEquals; +import static apoc.export.graphml.ExportGraphMLTestUtil.setUpGraphMl; import static apoc.util.BinaryTestUtil.getDecompressedData; import static apoc.util.BinaryTestUtil.fileToBinary; import static apoc.util.MapUtil.map; @@ -54,7 +58,6 @@ import static org.neo4j.configuration.GraphDatabaseSettings.TransactionStateMemoryAllocation.OFF_HEAP; import static org.neo4j.configuration.SettingValueParsers.BYTES; import static org.neo4j.graphdb.Label.label; -import static org.xmlunit.diff.ElementSelectors.byName; /** * @author mh @@ -62,131 +65,9 @@ */ public class ExportGraphMLTest { - public static final List ATTRIBUTES_CONTAINING_NODE_IDS = Arrays.asList("id", "source", "target"); - - public static final String KEY_TYPES_EMPTY = "%n" + - "%n" + - "%n"; - public static final String GRAPH = "%n"; - public static final String HEADER = "%n" + - "%n"; - public static final String KEY_TYPES_FALSE = "%n" + - "%n" + - "%n" + - "%n"+ - "%n" + - "%n" + - "%n"; - public static final String KEY_TYPES_FALSE_TINKER = "%n" + - "%n" + - "%n"+ - "%n" + - "%n" + - "" + - "%n"; - public static final String KEY_TYPES_DATA = "\n" + - ""; - public static final String KEY_TYPES = "%n" + - "%n" + - "%n" + - "%n"+ - "%n" + - "%n" + - "%n"; - public static final String KEY_TYPES_PATH = "%n" + - "%n" + - "%n"+ - "%n" + - "%n" + - "%n" + - "%n" + - "%n"; - public static final String KEY_TYPES_PATH_TINKERPOP = "%n" + - "%n" + - "%n"+ - "%n" + - "%n" + - "%n"; - public static final String KEY_TYPES_CAMEL_CASE = "%n" + - "%n" + - "%n" + - "%n" + - "%n" + - "%n" + - "%n"; - public static final String KEY_TYPES_NO_DATA_KEY = "\n" + - ""; - public static final String DATA = ":Foo:Foo0:Foo2{\"crs\":\"wgs-84-3d\",\"latitude\":12.78,\"longitude\":56.7,\"height\":100.0}foo2018-10-10%n" + - ":Bar42bar{\"crs\":\"wgs-84\",\"latitude\":12.78,\"longitude\":56.7,\"height\":null}%n" + - ":Bar12[1,2,3]%n" + - "KNOWS%n"; - public static final String DATA_TINKER = "Foo:Foo0:Foo2{\"crs\":\"wgs-84-3d\",\"latitude\":12.78,\"longitude\":56.7,\"height\":100.0}foo2018-10-10%n" + - "Bar42bar{\"crs\":\"wgs-84\",\"latitude\":12.78,\"longitude\":56.7,\"height\":null}%n" + - "Bar12[1,2,3]%n" + - "KNOWS%n"; - public static final String DATA_CAMEL_CASE = - ":Foo:Foo0:Foo2foofoo%n" + - ":Barbarbar42%n" + - "KNOWSKNOWS%n"; - - public static final String DATA_NODE_EDGE = " :FOOfoo %n" + - " :BARbar [a,b,c] %n" + - " :EDGE_LABEL foo %n" + - "TEST %n" + - " :QWERTYqwerty %n" + - " KNOWS %n"; - - private static final String DATA_WITHOUT_CHAR_DATA_KEYS = ":Foo:Foo0:Foo2{\"crs\":\"wgs-84-3d\",\"latitude\":12.78,\"longitude\":56.7,\"height\":100.0}foo2018-10-10%n" + - ":Bar42bar{\"crs\":\"wgs-84\",\"latitude\":12.78,\"longitude\":56.7,\"height\":null}%n" + - ":Bar12[1,2,3]%n" + - "\n"; - - public static final String DATA_NO_DATA_KEY = "C:\\bright\\itecembed\\obj\\ada\\a3_status.ads\n" + - "C:\\bright\\itecembed\\obj\\ada\\b3_status.ads\n" + - ""; - - public static final String FOOTER = "%n" + - ""; - - public static final String DATA_PATH = ":Foo:Foo0:Foo2foo{\"crs\":\"wgs-84-3d\",\"latitude\":12.78,\"longitude\":56.7,\"height\":100.0}foo2018-10-10%n" + - ":Barbar42bar{\"crs\":\"wgs-84\",\"latitude\":12.78,\"longitude\":56.7,\"height\":null}%n" + - "KNOWSKNOWS%n"; - - public static final String DATA_PATH_CAPTION = ":Foo:Foo0:Foo2foo{\"crs\":\"wgs-84-3d\",\"latitude\":12.78,\"longitude\":56.7,\"height\":100.0}foo2018-10-10%n" + - ":Barbar42bar{\"crs\":\"wgs-84\",\"latitude\":12.78,\"longitude\":56.7,\"height\":null}%n" + - "KNOWSKNOWS%n"; - - public static final String DATA_PATH_CAPTION_TINKER = "Foo:Foo0:Foo2{\"crs\":\"wgs-84-3d\",\"latitude\":12.78,\"longitude\":56.7,\"height\":100.0}foo2018-10-10%n" + - "Bar42bar{\"crs\":\"wgs-84\",\"latitude\":12.78,\"longitude\":56.7,\"height\":null}%n" + - "KNOWS%n"; - - public static final String DATA_PATH_CAPTION_DEFAULT = ":Foo:Foo0:Foo2point({x: 56.7, y: 12.78, z: 100.0, crs: 'wgs-84-3d'}){\"crs\":\"wgs-84-3d\",\"latitude\":12.78,\"longitude\":56.7,\"height\":100.0}foo2018-10-10%n" + - ":Barpoint({x: 56.7, y: 12.78, crs: 'wgs-84'})42bar{\"crs\":\"wgs-84\",\"latitude\":12.78,\"longitude\":56.7,\"height\":null}%n" + - "KNOWSKNOWS%n"; - - public static final String DATA_DATA = ":PersonFoo\n" + - ":PersonFoo0\n"; - - private static final String EXPECTED_TYPES_PATH = String.format(HEADER + KEY_TYPES_PATH + GRAPH + DATA_PATH + FOOTER); - private static final String EXPECTED_TYPES_PATH_CAPTION = String.format(HEADER + KEY_TYPES_PATH + GRAPH + DATA_PATH_CAPTION + FOOTER); - private static final String EXPECTED_TYPES_PATH_CAPTION_TINKER = String.format(HEADER + KEY_TYPES_PATH_TINKERPOP + GRAPH + DATA_PATH_CAPTION_TINKER + FOOTER); - private static final String EXPECTED_TYPES_PATH_WRONG_CAPTION = String.format(HEADER + KEY_TYPES_PATH + GRAPH + DATA_PATH_CAPTION_DEFAULT + FOOTER); - private static final String EXPECTED_TYPES = String.format(HEADER + KEY_TYPES + GRAPH + DATA + FOOTER); - private static final String EXPECTED_TYPES_WITHOUT_CHAR_DATA_KEYS = String.format(HEADER + KEY_TYPES + GRAPH + DATA_WITHOUT_CHAR_DATA_KEYS + FOOTER); - private static final String EXPECTED_FALSE = String.format(HEADER + KEY_TYPES_FALSE + GRAPH + DATA + FOOTER); - private static final String EXPECTED_TINKER = String.format(HEADER + KEY_TYPES_FALSE_TINKER + GRAPH + DATA_TINKER + FOOTER); - private static final String EXPECTED_DATA = String.format(HEADER + KEY_TYPES_DATA + GRAPH + DATA_DATA + FOOTER); - private static final String EXPECTED_READ_NODE_EDGE = String.format(HEADER + GRAPH + DATA_NODE_EDGE + FOOTER); - private static final String EXPECTED_TYPES_PATH_CAMEL_CASE = String.format(HEADER + KEY_TYPES_CAMEL_CASE + GRAPH + DATA_CAMEL_CASE + FOOTER); - private static final String DATA_EMPTY = ":Test3%n"; - private static final String EXPECTED_TYPES_EMPTY = String.format(HEADER + KEY_TYPES_EMPTY + GRAPH + DATA_EMPTY + FOOTER); - private static final String EXPECTED_TYPES_NO_DATA_KEY = String.format(HEADER + KEY_TYPES_NO_DATA_KEY + GRAPH + DATA_NO_DATA_KEY + FOOTER); - @Rule public TestName testName = new TestName(); - private static final String TEST_WITH_NO_IMPORT = "WithNoImportConfig"; - private static final String TEST_WITH_NO_EXPORT = "WithNoExportConfig"; @Rule public DbmsRule db = new ImpermanentDbmsRule() @@ -203,13 +84,7 @@ public class ExportGraphMLTest { @Before public void setUp() { - TestUtil.registerProcedure(db, ExportGraphML.class, Graphs.class); - - apocConfig().setProperty(APOC_EXPORT_FILE_ENABLED, Boolean.toString(!testName.getMethodName().endsWith(TEST_WITH_NO_EXPORT))); - apocConfig().setProperty(APOC_IMPORT_FILE_ENABLED, Boolean.toString(!testName.getMethodName().endsWith(TEST_WITH_NO_IMPORT))); - apocConfig().setProperty(APOC_IMPORT_FILE_USE_NEO4J_CONFIG, false); - - db.executeTransactionally("CREATE (f:Foo:Foo2:Foo0 {name:'foo', born:Date('2018-10-10'), place:point({ longitude: 56.7, latitude: 12.78, height: 100 })})-[:KNOWS]->(b:Bar {name:'bar',age:42, place:point({ longitude: 56.7, latitude: 12.78})}),(c:Bar {age:12,values:[1,2,3]})"); + setUpGraphMl(db, testName); } @Test @@ -616,35 +491,6 @@ public void testExportGraphGraphML() { assertXMLEquals(output, EXPECTED_FALSE); } - private void assertXMLEquals(Object output, String xmlString) { - Diff myDiff = DiffBuilder.compare(xmlString) - .withTest(output) - .checkForSimilar() - .ignoreWhitespace() - .withAttributeFilter(attr -> !ATTRIBUTES_CONTAINING_NODE_IDS.contains(attr.getLocalName())) // ignore id properties -// .withNodeMatcher(new DefaultNodeMatcher(ElementSelectors.byNameAndAllAttributes)) - // similar to ElementSelectors.byNameAndAllAttributes, but ignore blacklistes attributes - .withNodeMatcher(new DefaultNodeMatcher((ElementSelector) (controlElement, testElement) -> { - if (!byName.canBeCompared(controlElement, testElement)) { - return false; - } - Map cAttrs = Nodes.getAttributes(controlElement); - Map tAttrs = Nodes.getAttributes(testElement); - if (cAttrs.size() != tAttrs.size()) { - return false; - } - for (Map.Entry e: cAttrs.entrySet()) { - if ((!ATTRIBUTES_CONTAINING_NODE_IDS.contains(e.getKey().getLocalPart())) - && (!e.getValue().equals(tAttrs.get(e.getKey())))) { - return false; - } - } - return true; - })) - .build(); - - assertFalse(myDiff.toString(), myDiff.hasDifferences()); - } @Test public void testExportGraphGraphMLTypes() { diff --git a/core/src/test/java/apoc/export/graphml/ExportGraphMLTestUtil.java b/core/src/test/java/apoc/export/graphml/ExportGraphMLTestUtil.java new file mode 100644 index 000000000..94e995cd6 --- /dev/null +++ b/core/src/test/java/apoc/export/graphml/ExportGraphMLTestUtil.java @@ -0,0 +1,188 @@ +package apoc.export.graphml; + +import apoc.graph.Graphs; +import apoc.util.TestUtil; +import org.junit.rules.TestName; +import org.neo4j.graphdb.GraphDatabaseService; +import org.xmlunit.builder.DiffBuilder; +import org.xmlunit.diff.DefaultNodeMatcher; +import org.xmlunit.diff.Diff; +import org.xmlunit.diff.ElementSelector; +import org.xmlunit.util.Nodes; + +import javax.xml.namespace.QName; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import static apoc.ApocConfig.APOC_EXPORT_FILE_ENABLED; +import static apoc.ApocConfig.APOC_IMPORT_FILE_ENABLED; +import static apoc.ApocConfig.APOC_IMPORT_FILE_USE_NEO4J_CONFIG; +import static apoc.ApocConfig.apocConfig; +import static org.junit.Assert.assertFalse; +import static org.xmlunit.diff.ElementSelectors.byName; + +public class ExportGraphMLTestUtil { + private static final String KEY_TYPES_EMPTY = "%n" + + "%n" + + "%n"; + private static final String GRAPH = "%n"; + private static final String HEADER = "%n" + + "%n"; + private static final String KEY_TYPES_FALSE = "%n" + + "%n" + + "%n" + + "%n"+ + "%n" + + "%n" + + "%n"; + private static final String KEY_TYPES_FALSE_TINKER = "%n" + + "%n" + + "%n"+ + "%n" + + "%n" + + "" + + "%n"; + private static final String KEY_TYPES_DATA = "\n" + + ""; + private static final String KEY_TYPES = "%n" + + "%n" + + "%n" + + "%n"+ + "%n" + + "%n" + + "%n"; + private static final String KEY_TYPES_PATH = "%n" + + "%n" + + "%n"+ + "%n" + + "%n" + + "%n" + + "%n" + + "%n"; + private static final String KEY_TYPES_PATH_TINKERPOP = "%n" + + "%n" + + "%n"+ + "%n" + + "%n" + + "%n"; + private static final String KEY_TYPES_CAMEL_CASE = "%n" + + "%n" + + "%n" + + "%n" + + "%n" + + "%n" + + "%n"; + private static final String KEY_TYPES_NO_DATA_KEY = "\n" + + ""; + private static final String DATA = ":Foo:Foo0:Foo2{\"crs\":\"wgs-84-3d\",\"latitude\":12.78,\"longitude\":56.7,\"height\":100.0}foo2018-10-10%n" + + ":Bar42bar{\"crs\":\"wgs-84\",\"latitude\":12.78,\"longitude\":56.7,\"height\":null}%n" + + ":Bar12[1,2,3]%n" + + "KNOWS%n"; + private static final String DATA_TINKER = "Foo:Foo0:Foo2{\"crs\":\"wgs-84-3d\",\"latitude\":12.78,\"longitude\":56.7,\"height\":100.0}foo2018-10-10%n" + + "Bar42bar{\"crs\":\"wgs-84\",\"latitude\":12.78,\"longitude\":56.7,\"height\":null}%n" + + "Bar12[1,2,3]%n" + + "KNOWS%n"; + private static final String DATA_CAMEL_CASE = + ":Foo:Foo0:Foo2foofoo%n" + + ":Barbarbar42%n" + + "KNOWSKNOWS%n"; + + private static final String DATA_NODE_EDGE = " :FOOfoo %n" + + " :BARbar [a,b,c] %n" + + " :EDGE_LABEL foo %n" + + "TEST %n" + + " :QWERTYqwerty %n" + + " KNOWS %n"; + + private static final String DATA_WITHOUT_CHAR_DATA_KEYS = ":Foo:Foo0:Foo2{\"crs\":\"wgs-84-3d\",\"latitude\":12.78,\"longitude\":56.7,\"height\":100.0}foo2018-10-10%n" + + ":Bar42bar{\"crs\":\"wgs-84\",\"latitude\":12.78,\"longitude\":56.7,\"height\":null}%n" + + ":Bar12[1,2,3]%n" + + "\n"; + + private static final String DATA_NO_DATA_KEY = "C:\\bright\\itecembed\\obj\\ada\\a3_status.ads\n" + + "C:\\bright\\itecembed\\obj\\ada\\b3_status.ads\n" + + ""; + + private static final String FOOTER = "%n" + + ""; + + private static final String DATA_PATH = ":Foo:Foo0:Foo2foo{\"crs\":\"wgs-84-3d\",\"latitude\":12.78,\"longitude\":56.7,\"height\":100.0}foo2018-10-10%n" + + ":Barbar42bar{\"crs\":\"wgs-84\",\"latitude\":12.78,\"longitude\":56.7,\"height\":null}%n" + + "KNOWSKNOWS%n"; + + private static final String DATA_PATH_CAPTION = ":Foo:Foo0:Foo2foo{\"crs\":\"wgs-84-3d\",\"latitude\":12.78,\"longitude\":56.7,\"height\":100.0}foo2018-10-10%n" + + ":Barbar42bar{\"crs\":\"wgs-84\",\"latitude\":12.78,\"longitude\":56.7,\"height\":null}%n" + + "KNOWSKNOWS%n"; + + private static final String DATA_PATH_CAPTION_TINKER = "Foo:Foo0:Foo2{\"crs\":\"wgs-84-3d\",\"latitude\":12.78,\"longitude\":56.7,\"height\":100.0}foo2018-10-10%n" + + "Bar42bar{\"crs\":\"wgs-84\",\"latitude\":12.78,\"longitude\":56.7,\"height\":null}%n" + + "KNOWS%n"; + + private static final String DATA_PATH_CAPTION_DEFAULT = ":Foo:Foo0:Foo2point({x: 56.7, y: 12.78, z: 100.0, crs: 'wgs-84-3d'}){\"crs\":\"wgs-84-3d\",\"latitude\":12.78,\"longitude\":56.7,\"height\":100.0}foo2018-10-10%n" + + ":Barpoint({x: 56.7, y: 12.78, crs: 'wgs-84'})42bar{\"crs\":\"wgs-84\",\"latitude\":12.78,\"longitude\":56.7,\"height\":null}%n" + + "KNOWSKNOWS%n"; + + private static final String DATA_DATA = ":PersonFoo\n" + + ":PersonFoo0\n"; + + public static final String EXPECTED_TYPES_PATH = String.format(HEADER + KEY_TYPES_PATH + GRAPH + DATA_PATH + FOOTER); + public static final String EXPECTED_TYPES_PATH_CAPTION = String.format(HEADER + KEY_TYPES_PATH + GRAPH + DATA_PATH_CAPTION + FOOTER); + public static final String EXPECTED_TYPES_PATH_CAPTION_TINKER = String.format(HEADER + KEY_TYPES_PATH_TINKERPOP + GRAPH + DATA_PATH_CAPTION_TINKER + FOOTER); + public static final String EXPECTED_TYPES_PATH_WRONG_CAPTION = String.format(HEADER + KEY_TYPES_PATH + GRAPH + DATA_PATH_CAPTION_DEFAULT + FOOTER); + public static final String EXPECTED_TYPES = String.format(HEADER + KEY_TYPES + GRAPH + DATA + FOOTER); + public static final String EXPECTED_TYPES_WITHOUT_CHAR_DATA_KEYS = String.format(HEADER + KEY_TYPES + GRAPH + DATA_WITHOUT_CHAR_DATA_KEYS + FOOTER); + public static final String EXPECTED_FALSE = String.format(HEADER + KEY_TYPES_FALSE + GRAPH + DATA + FOOTER); + public static final String EXPECTED_TINKER = String.format(HEADER + KEY_TYPES_FALSE_TINKER + GRAPH + DATA_TINKER + FOOTER); + public static final String EXPECTED_DATA = String.format(HEADER + KEY_TYPES_DATA + GRAPH + DATA_DATA + FOOTER); + public static final String EXPECTED_READ_NODE_EDGE = String.format(HEADER + GRAPH + DATA_NODE_EDGE + FOOTER); + public static final String EXPECTED_TYPES_PATH_CAMEL_CASE = String.format(HEADER + KEY_TYPES_CAMEL_CASE + GRAPH + DATA_CAMEL_CASE + FOOTER); + public static final String DATA_EMPTY = ":Test3%n"; + public static final String EXPECTED_TYPES_EMPTY = String.format(HEADER + KEY_TYPES_EMPTY + GRAPH + DATA_EMPTY + FOOTER); + public static final String EXPECTED_TYPES_NO_DATA_KEY = String.format(HEADER + KEY_TYPES_NO_DATA_KEY + GRAPH + DATA_NO_DATA_KEY + FOOTER); + + + public static void assertXMLEquals(Object output, String xmlString) { + List attrsWithNodeIds = Arrays.asList("id", "source", "target"); + + Diff myDiff = DiffBuilder.compare(xmlString) + .withTest(output) + .checkForSimilar() + .ignoreWhitespace() + .withAttributeFilter(attr -> !attrsWithNodeIds.contains(attr.getLocalName())) // ignore id properties + // similar to ElementSelectors.byNameAndAllAttributes, but ignore blacklistes attributes + .withNodeMatcher(new DefaultNodeMatcher((ElementSelector) (controlElement, testElement) -> { + if (!byName.canBeCompared(controlElement, testElement)) { + return false; + } + Map cAttrs = Nodes.getAttributes(controlElement); + Map tAttrs = Nodes.getAttributes(testElement); + if (cAttrs.size() != tAttrs.size()) { + return false; + } + for (Map.Entry e: cAttrs.entrySet()) { + if ((!attrsWithNodeIds.contains(e.getKey().getLocalPart())) + && (!e.getValue().equals(tAttrs.get(e.getKey())))) { + return false; + } + } + return true; + })) + .build(); + + assertFalse(myDiff.toString(), myDiff.hasDifferences()); + } + + + public static void setUpGraphMl(GraphDatabaseService db, TestName testName) { + TestUtil.registerProcedure(db, ExportGraphML.class, Graphs.class); + + apocConfig().setProperty(APOC_EXPORT_FILE_ENABLED, + Boolean.toString(!testName.getMethodName().endsWith("WithNoImportConfig"))); + apocConfig().setProperty(APOC_IMPORT_FILE_ENABLED, + Boolean.toString(!testName.getMethodName().endsWith("WithNoExportConfig"))); + apocConfig().setProperty(APOC_IMPORT_FILE_USE_NEO4J_CONFIG, false); + + db.executeTransactionally("CREATE (f:Foo:Foo2:Foo0 {name:'foo', born:Date('2018-10-10'), place:point({ longitude: 56.7, latitude: 12.78, height: 100 })})-[:KNOWS]->(b:Bar {name:'bar',age:42, place:point({ longitude: 56.7, latitude: 12.78})}),(c:Bar {age:12,values:[1,2,3]})"); + } +} diff --git a/core/src/test/java/apoc/export/json/ExportJsonS3Test.java b/core/src/test/java/apoc/export/json/ExportJsonS3Test.java index 8ec6c2949..5fce2824a 100644 --- a/core/src/test/java/apoc/export/json/ExportJsonS3Test.java +++ b/core/src/test/java/apoc/export/json/ExportJsonS3Test.java @@ -1,78 +1,40 @@ package apoc.export.json; import apoc.graph.Graphs; -import apoc.util.JsonUtil; import apoc.util.TestUtil; -import apoc.util.s3.S3TestUtil; +import apoc.util.s3.S3BaseTest; import org.junit.Before; import org.junit.Rule; import org.junit.Test; -import org.junit.Ignore; -import org.neo4j.configuration.GraphDatabaseSettings; import org.neo4j.test.rule.DbmsRule; import org.neo4j.test.rule.ImpermanentDbmsRule; import java.io.File; -import java.io.IOException; -import java.nio.file.Paths; import java.util.Map; -import java.util.Optional; import static apoc.ApocConfig.APOC_EXPORT_FILE_ENABLED; import static apoc.ApocConfig.apocConfig; +import static apoc.util.FileTestUtil.assertStreamEquals; import static apoc.util.MapUtil.map; +import static apoc.util.s3.S3TestUtil.readS3FileToString; import static org.junit.Assert.*; -@Ignore("To use this test, you need to set the S3 bucket and region to a valid endpoint " + - "and have your access key and secret key setup in your environment.") -public class ExportJsonS3Test { - private static String S3_BUCKET_NAME = null; - - private static File directory = new File("target/import"); +public class ExportJsonS3Test extends S3BaseTest { private static File directoryExpected = new File("src/test/resources/exportJSON"); - static { //noinspection ResultOfMethodCallIgnored - directory.mkdirs(); - } - @Rule - public DbmsRule db = new ImpermanentDbmsRule() - .withSetting(GraphDatabaseSettings.load_csv_file_url_root, directory.toPath().toAbsolutePath()); - - private static String getS3Url(String key) { - return String.format("s3://:@/%s/%s", S3_BUCKET_NAME, key); - } - - private String readFile(String fileName) { - return TestUtil.readFileToString(new File(directory, fileName)); - } - - private void verifyUpload(String s3Url, String fileName) throws IOException { - S3TestUtil.readFile(s3Url, Paths.get(directory.toString(), fileName).toString()); - assertFileEquals(fileName); - } - - private static String getEnvVar(String envVarKey) throws Exception { - return Optional.ofNullable(System.getenv(envVarKey)).orElseThrow( - () -> new Exception(String.format("%s is not set in the environment", envVarKey)) - ); - } + public DbmsRule db = new ImpermanentDbmsRule(); @Before - public void setup() throws Exception { - if (S3_BUCKET_NAME == null) { - S3_BUCKET_NAME = getEnvVar("S3_BUCKET_NAME"); - } - - TestUtil.registerProcedure(db, ExportJson.class, Graphs.class); + public void setUp() throws Exception { apocConfig().setProperty(APOC_EXPORT_FILE_ENABLED, true); - db.executeTransactionally("CREATE (f:User {name:'Adam',age:42,male:true,kids:['Sam','Anna','Grace'], born:localdatetime('2015185T19:32:24'), place:point({latitude: 13.1, longitude: 33.46789})})-[:KNOWS {since: 1993, bffSince: duration('P5M1.5D')}]->(b:User {name:'Jim',age:42}),(c:User {age:12})"); + TestUtil.registerProcedure(db, ExportJson.class, Graphs.class); } @Test public void testExportAllJson() throws Exception { String filename = "all.json"; - String s3Url = getS3Url(filename); + String s3Url = s3Container.getUrl(filename); TestUtil.testCall(db, "CALL apoc.export.json.all($s3,null)", map("s3", s3Url), @@ -80,13 +42,13 @@ public void testExportAllJson() throws Exception { assertResults(s3Url, r, "database"); } ); - verifyUpload(s3Url, filename); + assertStreamStringEquals(directoryExpected, s3Url); } @Test public void testExportPointMapDatetimeJson() throws Exception { String filename = "mapPointDatetime.json"; - String s3Url = getS3Url(filename); + String s3Url = s3Container.getUrl(filename); String query = "return {data: 1, value: {age: 12, name:'Mike', data: {number: [1,3,5], born: date('2018-10-29'), place: point({latitude: 13.1, longitude: 33.46789})}}} as map, " + "datetime('2015-06-24T12:50:35.556+0100') AS theDateTime, " + "localdatetime('2015185T19:32:24') AS theLocalDateTime," + @@ -101,15 +63,14 @@ public void testExportPointMapDatetimeJson() throws Exception { assertTrue("Should get statement",r.get("source").toString().contains("statement: cols(7)")); assertEquals(s3Url, r.get("file")); assertEquals("json", r.get("format")); - assertFileEquals(filename); }); - verifyUpload(s3Url, filename); + assertStreamStringEquals(directoryExpected, s3Url); } @Test public void testExportListNode() throws Exception { String filename = "listNode.json"; - String s3Url = getS3Url(filename); + String s3Url = s3Container.getUrl(filename); String query = "MATCH (u:User) RETURN COLLECT(u) as list"; TestUtil.testCall(db, "CALL apoc.export.json.query($query,$s3)", @@ -119,13 +80,13 @@ public void testExportListNode() throws Exception { assertEquals(s3Url, r.get("file")); assertEquals("json", r.get("format")); }); - verifyUpload(s3Url, filename); + assertStreamStringEquals(directoryExpected, s3Url); } @Test public void testExportListRel() throws Exception { String filename = "listRel.json"; - String s3Url = getS3Url(filename); + String s3Url = s3Container.getUrl(filename); String query = "MATCH (u:User)-[rel:KNOWS]->(u2:User) RETURN COLLECT(rel) as list"; TestUtil.testCall(db, "CALL apoc.export.json.query($query,$s3)", @@ -135,13 +96,13 @@ public void testExportListRel() throws Exception { assertEquals(s3Url, r.get("file")); assertEquals("json", r.get("format")); }); - verifyUpload(s3Url, filename); + assertStreamStringEquals(directoryExpected, s3Url); } @Test public void testExportListPath() throws Exception { String filename = "listPath.json"; - String s3Url = getS3Url(filename); + String s3Url = s3Container.getUrl(filename); String query = "MATCH p = (u:User)-[rel]->(u2:User) RETURN COLLECT(p) as list"; TestUtil.testCall(db, "CALL apoc.export.json.query($query,$s3)", @@ -151,13 +112,13 @@ public void testExportListPath() throws Exception { assertEquals(s3Url, r.get("file")); assertEquals("json", r.get("format")); }); - verifyUpload(s3Url, filename); + assertStreamStringEquals(directoryExpected, s3Url); } @Test public void testExportMap() throws Exception { String filename = "MapNode.json"; - String s3Url = getS3Url(filename); + String s3Url = s3Container.getUrl(filename); String query = "MATCH (u:User)-[r:KNOWS]->(d:User) RETURN u {.*}, d {.*}, r {.*}"; TestUtil.testCall(db, "CALL apoc.export.json.query($query,$s3)", @@ -167,14 +128,14 @@ public void testExportMap() throws Exception { assertEquals(s3Url, r.get("file")); assertEquals("json", r.get("format")); }); - verifyUpload(s3Url, filename); + assertStreamStringEquals(directoryExpected, s3Url); } @Test public void testExportMapPath() throws Exception { db.executeTransactionally("CREATE (f:User {name:'Mike',age:78,male:true})-[:KNOWS {since: 1850}]->(b:User {name:'John',age:18}),(c:User {age:39})"); String filename = "MapPath.json"; - String s3Url = getS3Url(filename); + String s3Url = s3Container.getUrl(filename); String query = "MATCH path = (u:User)-[rel:KNOWS]->(u2:User) RETURN {key:path} as map, 'Kate' as name"; TestUtil.testCall(db, "CALL apoc.export.json.query($query,$s3)", @@ -184,13 +145,13 @@ public void testExportMapPath() throws Exception { assertEquals(s3Url, r.get("file")); assertEquals("json", r.get("format")); }); - verifyUpload(s3Url, filename); + assertStreamStringEquals(directoryExpected, s3Url); } @Test public void testExportMapRel() throws Exception { String filename = "MapRel.json"; - String s3Url = getS3Url(filename); + String s3Url = s3Container.getUrl(filename); String query = "MATCH p = (u:User)-[rel:KNOWS]->(u2:User) RETURN rel {.*}"; TestUtil.testCall(db, "CALL apoc.export.json.query($query,$s3)", @@ -200,13 +161,13 @@ public void testExportMapRel() throws Exception { assertEquals(s3Url, r.get("file")); assertEquals("json", r.get("format")); }); - verifyUpload(s3Url, filename); + assertStreamStringEquals(directoryExpected, s3Url); } @Test public void testExportMapComplex() throws Exception { String filename = "MapComplex.json"; - String s3Url = getS3Url(filename); + String s3Url = s3Container.getUrl(filename); String query = "RETURN {value:1, data:[10,'car',null, point({ longitude: 56.7, latitude: 12.78 }), point({ longitude: 56.7, latitude: 12.78, height: 8 }), point({ x: 2.3, y: 4.5 }), point({ x: 2.3, y: 4.5, z: 2 }),date('2018-10-10'), datetime('2018-10-18T14:21:40.004Z'), localdatetime({ year:1984, week:10, dayOfWeek:3, hour:12, minute:31, second:14, millisecond: 645 }), {x:1, y:[1,2,3,{age:10}]}]} as key"; TestUtil.testCall(db, "CALL apoc.export.json.query($query,$s3)", @@ -216,13 +177,13 @@ public void testExportMapComplex() throws Exception { assertEquals(s3Url, r.get("file")); assertEquals("json", r.get("format")); }); - verifyUpload(s3Url, filename); + assertStreamStringEquals(directoryExpected, s3Url); } @Test public void testExportGraphJson() throws Exception { String filename = "graph.json"; - String s3Url = getS3Url(filename); + String s3Url = s3Container.getUrl(filename); TestUtil.testCall(db, "CALL apoc.graph.fromDB('test',{}) yield graph " + "CALL apoc.export.json.graph(graph, $s3) " + @@ -230,13 +191,13 @@ public void testExportGraphJson() throws Exception { "RETURN *", map("s3", s3Url), (r) -> assertResults(s3Url, r, "graph")); - verifyUpload(s3Url, filename); + assertStreamStringEquals(directoryExpected, s3Url); } @Test public void testExportQueryJson() throws Exception { String filename = "query.json"; - String s3Url = getS3Url(filename); + String s3Url = s3Container.getUrl(filename); String query = "MATCH (u:User) return u.age, u.name, u.male, u.kids, labels(u)"; TestUtil.testCall(db, "CALL apoc.export.json.query($query,$s3)", @@ -246,13 +207,13 @@ public void testExportQueryJson() throws Exception { assertEquals(s3Url, r.get("file")); assertEquals("json", r.get("format")); }); - verifyUpload(s3Url, filename); + assertStreamStringEquals(directoryExpected, s3Url); } @Test public void testExportQueryNodesJson() throws Exception { String filename = "query_nodes.json"; - String s3Url = getS3Url(filename); + String s3Url = s3Container.getUrl(filename); String query = "MATCH (u:User) return u"; TestUtil.testCall(db, "CALL apoc.export.json.query($query,$s3)", @@ -262,13 +223,13 @@ public void testExportQueryNodesJson() throws Exception { assertEquals(s3Url, r.get("file")); assertEquals("json", r.get("format")); }); - verifyUpload(s3Url, filename); + assertStreamStringEquals(directoryExpected, s3Url); } @Test public void testExportQueryTwoNodesJson() throws Exception { String filename = "query_two_nodes.json"; - String s3Url = getS3Url(filename); + String s3Url = s3Container.getUrl(filename); String query = "MATCH (u:User{name:'Adam'}), (l:User{name:'Jim'}) return u, l"; TestUtil.testCall(db, "CALL apoc.export.json.query($query,$s3)", @@ -278,13 +239,13 @@ public void testExportQueryTwoNodesJson() throws Exception { assertEquals(s3Url, r.get("file")); assertEquals("json", r.get("format")); }); - verifyUpload(s3Url, filename); + assertStreamStringEquals(directoryExpected, s3Url); } @Test public void testExportQueryNodesJsonParams() throws Exception { String filename = "query_nodes_param.json"; - String s3Url = getS3Url(filename); + String s3Url = s3Container.getUrl(filename); String query = "MATCH (u:User) WHERE u.age > $age return u"; TestUtil.testCall(db, "CALL apoc.export.json.query($query,$s3,{params:{age:10}})", @@ -294,13 +255,13 @@ public void testExportQueryNodesJsonParams() throws Exception { assertEquals(s3Url, r.get("file")); assertEquals("json", r.get("format")); }); - verifyUpload(s3Url, filename); + assertStreamStringEquals(directoryExpected, s3Url); } @Test public void testExportQueryNodesJsonCount() throws Exception { String filename = "query_nodes_count.json"; - String s3Url = getS3Url(filename); + String s3Url = s3Container.getUrl(filename); String query = "MATCH (n) return count(n)"; TestUtil.testCall(db, "CALL apoc.export.json.query($query,$s3)", @@ -310,13 +271,13 @@ public void testExportQueryNodesJsonCount() throws Exception { assertEquals(s3Url, r.get("file")); assertEquals("json", r.get("format")); }); - verifyUpload(s3Url, filename); + assertStreamStringEquals(directoryExpected, s3Url); } @Test public void testExportData() throws Exception { String filename = "data.json"; - String s3Url = getS3Url(filename); + String s3Url = s3Container.getUrl(filename); TestUtil.testCall(db, "MATCH (nod:User) " + "MATCH ()-[reels:KNOWS]->() " + @@ -329,13 +290,13 @@ public void testExportData() throws Exception { assertEquals(s3Url, r.get("file")); assertEquals("json", r.get("format")); }); - verifyUpload(s3Url, filename); + assertStreamStringEquals(directoryExpected, s3Url); } @Test public void testExportDataPath() throws Exception { String filename = "query_nodes_path.json"; - String s3Url = getS3Url(filename); + String s3Url = s3Container.getUrl(filename); String query = "MATCH p = (u:User)-[rel]->(u2:User) return u, rel, u2, p, u.name"; TestUtil.testCall(db, "CALL apoc.export.json.query($query,$s3)", @@ -344,13 +305,13 @@ public void testExportDataPath() throws Exception { assertEquals(s3Url, r.get("file")); assertEquals("json", r.get("format")); }); - verifyUpload(s3Url, filename); + assertStreamStringEquals(directoryExpected, s3Url); } @Test public void testExportAllWithWriteNodePropertiesJson() throws Exception { String filename = "with_node_properties.json"; - String s3Url = getS3Url(filename); + String s3Url = s3Container.getUrl(filename); String query = "MATCH p = (u:User)-[rel:KNOWS]->(u2:User) RETURN rel"; TestUtil.testCall(db, "CALL apoc.export.json.query($query,$s3,{writeNodeProperties:true})", @@ -360,13 +321,13 @@ public void testExportAllWithWriteNodePropertiesJson() throws Exception { assertEquals(s3Url, r.get("file")); assertEquals("json", r.get("format")); }); - verifyUpload(s3Url, filename); + assertStreamStringEquals(directoryExpected, s3Url); } @Test public void testExportAllWithDefaultWriteNodePropertiesJson() throws Exception { String filename = "with_node_properties.json"; - String s3Url = getS3Url(filename); + String s3Url = s3Container.getUrl(filename); String query = "MATCH p = (u:User)-[rel:KNOWS]->(u2:User) RETURN rel"; // default value for writeNodeProperties is true @@ -377,13 +338,13 @@ public void testExportAllWithDefaultWriteNodePropertiesJson() throws Exception { assertEquals(s3Url, r.get("file")); assertEquals("json", r.get("format")); }); - verifyUpload(s3Url, filename); + assertStreamStringEquals(directoryExpected, s3Url); } @Test public void testExportAllWithoutWriteNodePropertiesJson() throws Exception { String filename = "without_node_properties.json"; - String s3Url = getS3Url(filename); + String s3Url = s3Container.getUrl(filename); String query = "MATCH p = (u:User)-[rel:KNOWS]->(u2:User) RETURN rel"; TestUtil.testCall(db, "CALL apoc.export.json.query($query,$s3,{writeNodeProperties:false})", @@ -393,14 +354,14 @@ public void testExportAllWithoutWriteNodePropertiesJson() throws Exception { assertEquals(s3Url, r.get("file")); assertEquals("json", r.get("format")); }); - verifyUpload(s3Url, filename); + assertStreamStringEquals(directoryExpected, s3Url); } @Test public void testExportQueryOrderJson() throws Exception { db.executeTransactionally("CREATE (f:User12:User1:User0:User {name:'Alan'})"); String filename = "query_node_labels.json"; - String s3Url = getS3Url(filename); + String s3Url = s3Container.getUrl(filename); String query = "MATCH (u:User) WHERE u.name='Alan' RETURN u"; TestUtil.testCall(db, "CALL apoc.export.json.query($query,$s3)", @@ -410,7 +371,12 @@ public void testExportQueryOrderJson() throws Exception { assertEquals(s3Url, r.get("file")); assertEquals("json", r.get("format")); }); - verifyUpload(s3Url, filename); + assertStreamStringEquals(directoryExpected, s3Url); + } + + private void assertStreamStringEquals(File directoryExpected, String s3Url) { + final String actual = readS3FileToString(s3Url); + assertStreamEquals(directoryExpected, s3Url, actual); } private void assertResults(String filename, Map r, final String source) { @@ -422,19 +388,4 @@ private void assertResults(String filename, Map r, final String assertEquals("json", r.get("format")); assertTrue("Should get time greater than 0",((long) r.get("time")) >= 0); } - - private void assertFileEquals(String fileName) { - String actualText = TestUtil.readFileToString(new File(directory, fileName)); - assertStreamEquals(fileName, actualText); - } - - private void assertStreamEquals(String fileName, String actualText) { - String expectedText = TestUtil.readFileToString(new File(directoryExpected, fileName)); - String[] actualArray = actualText.split("\n"); - String[] expectArray = expectedText.split("\n"); - assertEquals(expectArray.length, actualArray.length); - for (int i = 0; i < actualArray.length; i++) { - assertEquals(JsonUtil.parse(expectArray[i],null, Object.class), JsonUtil.parse(actualArray[i],null, Object.class)); - } - } } diff --git a/core/src/test/java/apoc/export/json/ExportJsonTest.java b/core/src/test/java/apoc/export/json/ExportJsonTest.java index 10ab18933..dbdb038b0 100644 --- a/core/src/test/java/apoc/export/json/ExportJsonTest.java +++ b/core/src/test/java/apoc/export/json/ExportJsonTest.java @@ -3,7 +3,7 @@ import apoc.graph.Graphs; import apoc.util.BinaryTestUtil; import apoc.util.CompressionAlgo; -import apoc.util.JsonUtil; +import apoc.util.FileTestUtil; import apoc.util.TestUtil; import apoc.util.Util; import org.junit.Before; @@ -563,13 +563,7 @@ private void assertStreamResults(Map r, final String source) { assertTrue("Should get time greater than 0",((long) r.get("time")) >= 0); } - private void assertStreamEquals(String fileName, String actualText) { - String expectedText = TestUtil.readFileToString(new File(directoryExpected, fileName)); - String[] actualArray = actualText.split("\n"); - String[] expectArray = expectedText.split("\n"); - assertEquals(expectArray.length, actualArray.length); - for (int i = 0; i < actualArray.length; i++) { - assertEquals(JsonUtil.parse(expectArray[i],null, Object.class), JsonUtil.parse(actualArray[i],null, Object.class)); - } + private void assertStreamEquals(String fileName, String actualText) { + FileTestUtil.assertStreamEquals(directory, fileName, actualText); } } diff --git a/test-utils/build.gradle b/test-utils/build.gradle index 8c3adf515..9d424f97b 100644 --- a/test-utils/build.gradle +++ b/test-utils/build.gradle @@ -44,6 +44,7 @@ dependencies { implementation group: 'org.neo4j', name: 'neo4j-io', version: neo4jVersionEffective, classifier: "tests" implementation group: 'org.gradle', name: 'gradle-tooling-api', version: '7.2' implementation group: 'org.jetbrains', name: 'annotations', version: "17.0.0" + implementation group: 'com.amazonaws', name: 'aws-java-sdk-s3', version: '1.11.270' } diff --git a/test-utils/src/main/java/apoc/util/FileTestUtil.java b/test-utils/src/main/java/apoc/util/FileTestUtil.java new file mode 100644 index 000000000..7b24c3ee8 --- /dev/null +++ b/test-utils/src/main/java/apoc/util/FileTestUtil.java @@ -0,0 +1,18 @@ +package apoc.util; + +import java.io.File; + +import static org.junit.Assert.assertEquals; + +public class FileTestUtil { + + public static void assertStreamEquals(File directoryExpected, String fileName, String actualText) { + String expectedText = TestUtil.readFileToString(new File(directoryExpected, fileName)); + String[] actualArray = actualText.split("\n"); + String[] expectArray = expectedText.split("\n"); + assertEquals(expectArray.length, actualArray.length); + for (int i = 0; i < actualArray.length; i++) { + assertEquals(JsonUtil.parse(expectArray[i],null, Object.class), JsonUtil.parse(actualArray[i],null, Object.class)); + } + } +} diff --git a/test-utils/src/main/java/apoc/util/s3/S3BaseTest.java b/test-utils/src/main/java/apoc/util/s3/S3BaseTest.java new file mode 100644 index 000000000..3538b6e5d --- /dev/null +++ b/test-utils/src/main/java/apoc/util/s3/S3BaseTest.java @@ -0,0 +1,25 @@ +package apoc.util.s3; + +import org.junit.AfterClass; +import org.junit.BeforeClass; + +public abstract class S3BaseTest { + protected static S3Container s3Container; + + @BeforeClass + public static void baseBeforeClass() { + s3Container = new S3Container(); + + // In test environment we skip the MD5 validation that can cause issues + System.setProperty("com.amazonaws.services.s3.disableGetObjectMD5Validation", "true"); + System.setProperty("com.amazonaws.sdk.disableCertChecking", "true"); + } + + @AfterClass + public static void tearDown() { + System.clearProperty("com.amazonaws.services.s3.disableGetObjectMD5Validation"); + System.clearProperty("com.amazonaws.sdk.disableCertChecking"); + + s3Container.close(); + } +} diff --git a/test-utils/src/main/java/apoc/util/s3/S3Container.java b/test-utils/src/main/java/apoc/util/s3/S3Container.java new file mode 100644 index 000000000..3b53e3f4b --- /dev/null +++ b/test-utils/src/main/java/apoc/util/s3/S3Container.java @@ -0,0 +1,65 @@ +package apoc.util.s3; + +import apoc.util.Util; +import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.client.builder.AwsClientBuilder; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import org.testcontainers.containers.localstack.LocalStackContainer; +import org.testcontainers.utility.DockerImageName; + +import java.io.File; + +import static org.junit.Assert.assertTrue; +import static org.testcontainers.containers.localstack.LocalStackContainer.Service.S3; + +public class S3Container implements AutoCloseable { + private final static String S3_BUCKET_NAME = "test-bucket"; + private final LocalStackContainer localstack; + private final AmazonS3 s3; + + public S3Container() { + localstack = new LocalStackContainer(DockerImageName.parse("localstack/localstack:1.2.0")) + .withServices(S3); + localstack.start(); + + s3 = AmazonS3ClientBuilder + .standard() + .withEndpointConfiguration(getEndpointConfiguration()) + .withCredentials(getCredentialsProvider()) + .build(); + s3.createBucket(S3_BUCKET_NAME); + + assertTrue("Localstack container not correctly started. The provided logs are: \n" + localstack.getLogs(), + localstack.isRunning()); + } + + public void close() { + Util.close(localstack); + } + + public AwsClientBuilder.EndpointConfiguration getEndpointConfiguration() { + return localstack.getEndpointConfiguration(S3); + } + + public AWSCredentialsProvider getCredentialsProvider() { + return localstack.getDefaultCredentialsProvider(); + } + + public String getUrl(String key) { + return String.format("s3://%s.%s/%s/%s?accessKey=%s&secretKey=%s", + localstack.getEndpointConfiguration(S3).getSigningRegion(), + localstack.getEndpointConfiguration(S3).getServiceEndpoint() + .replace("http://", ""), + S3_BUCKET_NAME, + key, + localstack.getDefaultCredentialsProvider().getCredentials().getAWSAccessKeyId(), + localstack.getDefaultCredentialsProvider().getCredentials().getAWSSecretKey()); + } + + public String putFile(String fileName) { + final File file = new File(fileName); + s3.putObject(S3_BUCKET_NAME, file.getName(), file); + return getUrl(file.getName()); + } +} diff --git a/common/src/test/java/apoc/util/s3/S3ParamsExtractorTest.java b/test-utils/src/main/java/apoc/util/s3/S3ParamsExtractorTest.java similarity index 100% rename from common/src/test/java/apoc/util/s3/S3ParamsExtractorTest.java rename to test-utils/src/main/java/apoc/util/s3/S3ParamsExtractorTest.java diff --git a/test-utils/src/main/java/apoc/util/s3/S3TestUtil.java b/test-utils/src/main/java/apoc/util/s3/S3TestUtil.java new file mode 100644 index 000000000..251a352bd --- /dev/null +++ b/test-utils/src/main/java/apoc/util/s3/S3TestUtil.java @@ -0,0 +1,47 @@ +package apoc.util.s3; + +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.model.S3Object; +import com.amazonaws.services.s3.model.S3ObjectInputStream; +import com.amazonaws.util.IOUtils; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; + +import static org.junit.Assert.assertEquals; + +/** + * Utility class for testing Amazon S3 related functionality. + */ +public class S3TestUtil { + + /** + * Read file object as a string from S3 bucket. This code expects valid AWS credentials are set up. + * @param s3Url String containing url to S3 bucket. + * @return the s3 string object + */ + public static String readS3FileToString(String s3Url) { + try { + S3Object s3object = getS3Object(s3Url); + S3ObjectInputStream inputStream = s3object.getObjectContent(); + return IOUtils.toString(inputStream); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public static S3Object getS3Object(String s3Url) throws MalformedURLException { + S3Params s3Params = S3ParamsExtractor.extract(new URL(s3Url)); + S3Aws s3Aws = new S3Aws(s3Params, s3Params.getRegion()); + AmazonS3 s3Client = s3Aws.getClient(); + + S3Object s3object = s3Client.getObject(s3Params.getBucket(), s3Params.getKey()); + return s3object; + } + + public static void assertStringFileEquals(String expected, String s3Url) { + final String actual = readS3FileToString(s3Url); + assertEquals(expected, actual); + } +} From 5d648abc8b87905401faa18616727ce1ae4fb5ee Mon Sep 17 00:00:00 2001 From: Giuseppe Villani Date: Fri, 18 Nov 2022 10:38:56 +0100 Subject: [PATCH 2/3] fix tests --- .../export/cypher/ExportCypherS3Test.java | 42 +--------- .../apoc/export/cypher/ExportCypherTest.java | 82 ++++++++++--------- .../export/graphml/ExportGraphMLTestUtil.java | 4 +- .../apoc/export/json/ExportJsonS3Test.java | 47 +++++------ .../java/apoc/export/json/ExportJsonTest.java | 2 +- 5 files changed, 70 insertions(+), 107 deletions(-) diff --git a/core/src/test/java/apoc/export/cypher/ExportCypherS3Test.java b/core/src/test/java/apoc/export/cypher/ExportCypherS3Test.java index b904c6287..5536dcb8b 100644 --- a/core/src/test/java/apoc/export/cypher/ExportCypherS3Test.java +++ b/core/src/test/java/apoc/export/cypher/ExportCypherS3Test.java @@ -17,7 +17,6 @@ import static apoc.ApocConfig.APOC_EXPORT_FILE_ENABLED; import static apoc.ApocConfig.apocConfig; import static apoc.export.cypher.ExportCypherTest.ExportCypherResults.*; -import static apoc.export.util.ExportFormat.*; import static apoc.util.Util.map; import static apoc.util.s3.S3TestUtil.assertStringFileEquals; import static org.junit.Assert.*; @@ -480,46 +479,7 @@ public void testExportQueryCypherShellUnwindBatchParamsWithOddBatchSizeOddDatase map("s3", s3Url), (r) -> {}); db.executeTransactionally("MATCH (n:Bar {name:'bar3',age:35}), (n1:Bar {name:'bar4',age:36}) DELETE n, n1"); - String expectedNodes = String.format(":begin%n" + - ":param rows => [{_id:4, properties:{age:12}}, {_id:5, properties:{age:4}}]%n" + - "UNWIND $rows AS row%n" + - "CREATE (n:`UNIQUE IMPORT LABEL`{`UNIQUE IMPORT ID`: row._id}) SET n += row.properties SET n:Bar;%n" + - ":param rows => [{_id:0, properties:{born:date('2018-10-31'), name:\"foo\"}}]%n" + - "UNWIND $rows AS row%n" + - "CREATE (n:`UNIQUE IMPORT LABEL`{`UNIQUE IMPORT ID`: row._id}) SET n += row.properties SET n:Foo;%n" + - ":commit%n" + - ":begin%n" + - ":param rows => [{_id:1, properties:{born:date('2017-09-29'), name:\"foo2\"}}, {_id:2, properties:{born:date('2016-03-12'), name:\"foo3\"}}]%n" + - "UNWIND $rows AS row%n" + - "CREATE (n:`UNIQUE IMPORT LABEL`{`UNIQUE IMPORT ID`: row._id}) SET n += row.properties SET n:Foo;%n" + - ":param rows => [{name:\"bar\", properties:{age:42}}]%n" + - "UNWIND $rows AS row%n" + - "CREATE (n:Bar{name: row.name}) SET n += row.properties;%n" + - ":commit%n" + - ":begin%n" + - ":param rows => [{name:\"bar2\", properties:{age:44}}, {name:\"bar3\", properties:{age:35}}]%n" + - "UNWIND $rows AS row%n" + - "CREATE (n:Bar{name: row.name}) SET n += row.properties;%n" + - ":param rows => [{name:\"bar4\", properties:{age:36}}]%n" + - "UNWIND $rows AS row%n" + - "CREATE (n:Bar{name: row.name}) SET n += row.properties;%n" + - ":commit%n"); - int expectedDropNum = 3; - String expectedDrop = String.format(":begin%n" + - "MATCH (n:`UNIQUE IMPORT LABEL`) WITH n LIMIT %d REMOVE n:`UNIQUE IMPORT LABEL` REMOVE n.`UNIQUE IMPORT ID`;%n" + - ":commit%n" + - ":begin%n" + - "MATCH (n:`UNIQUE IMPORT LABEL`) WITH n LIMIT %d REMOVE n:`UNIQUE IMPORT LABEL` REMOVE n.`UNIQUE IMPORT ID`;%n" + - ":commit%n" + - ":begin%n" + - "DROP CONSTRAINT uniqueConstraint;%n" + - ":commit%n", expectedDropNum, expectedDropNum); - String expected = (EXPECTED_SCHEMA + expectedNodes + EXPECTED_RELATIONSHIPS_PARAMS_ODD + expectedDrop) - .replace(NEO4J_SHELL.begin(), CYPHER_SHELL.begin()) - .replace(NEO4J_SHELL.commit(), CYPHER_SHELL.commit()) - .replace(NEO4J_SHELL.schemaAwait(), EXPECTED_INDEXES_AWAIT) - .replace(NEO4J_SHELL.schemaAwait(), CYPHER_SHELL.schemaAwait()); - assertStringFileEquals(expected, s3Url); + assertStringFileEquals(EXPECTED_QUERY_PARAMS_ODD, s3Url); } private void getUrlAndAssertEquals(String expected, String fileName) { diff --git a/core/src/test/java/apoc/export/cypher/ExportCypherTest.java b/core/src/test/java/apoc/export/cypher/ExportCypherTest.java index 83ffd3e91..b4739af62 100644 --- a/core/src/test/java/apoc/export/cypher/ExportCypherTest.java +++ b/core/src/test/java/apoc/export/cypher/ExportCypherTest.java @@ -605,46 +605,7 @@ public void testExportQueryCypherShellUnwindBatchParamsWithOddBatchSizeOddDatase map("file", fileName), (r) -> {}); db.executeTransactionally("MATCH (n:Bar {name:'bar3',age:35}), (n1:Bar {name:'bar4',age:36}) DELETE n, n1"); - String expectedNodes = String.format(":begin%n" + - ":param rows => [{_id:4, properties:{age:12}}, {_id:5, properties:{age:4}}]%n" + - "UNWIND $rows AS row%n" + - "CREATE (n:`UNIQUE IMPORT LABEL`{`UNIQUE IMPORT ID`: row._id}) SET n += row.properties SET n:Bar;%n" + - ":param rows => [{_id:0, properties:{born:date('2018-10-31'), name:\"foo\"}}]%n" + - "UNWIND $rows AS row%n" + - "CREATE (n:`UNIQUE IMPORT LABEL`{`UNIQUE IMPORT ID`: row._id}) SET n += row.properties SET n:Foo;%n" + - ":commit%n" + - ":begin%n" + - ":param rows => [{_id:1, properties:{born:date('2017-09-29'), name:\"foo2\"}}, {_id:2, properties:{born:date('2016-03-12'), name:\"foo3\"}}]%n" + - "UNWIND $rows AS row%n" + - "CREATE (n:`UNIQUE IMPORT LABEL`{`UNIQUE IMPORT ID`: row._id}) SET n += row.properties SET n:Foo;%n" + - ":param rows => [{name:\"bar\", properties:{age:42}}]%n" + - "UNWIND $rows AS row%n" + - "CREATE (n:Bar{name: row.name}) SET n += row.properties;%n" + - ":commit%n" + - ":begin%n" + - ":param rows => [{name:\"bar2\", properties:{age:44}}, {name:\"bar3\", properties:{age:35}}]%n" + - "UNWIND $rows AS row%n" + - "CREATE (n:Bar{name: row.name}) SET n += row.properties;%n" + - ":param rows => [{name:\"bar4\", properties:{age:36}}]%n" + - "UNWIND $rows AS row%n" + - "CREATE (n:Bar{name: row.name}) SET n += row.properties;%n" + - ":commit%n"); - int expectedDropNum = 3; - String expectedDrop = String.format(":begin%n" + - "MATCH (n:`UNIQUE IMPORT LABEL`) WITH n LIMIT %d REMOVE n:`UNIQUE IMPORT LABEL` REMOVE n.`UNIQUE IMPORT ID`;%n" + - ":commit%n" + - ":begin%n" + - "MATCH (n:`UNIQUE IMPORT LABEL`) WITH n LIMIT %d REMOVE n:`UNIQUE IMPORT LABEL` REMOVE n.`UNIQUE IMPORT ID`;%n" + - ":commit%n" + - ":begin%n" + - "DROP CONSTRAINT UNIQUE_IMPORT_NAME;%n" + - ":commit%n", expectedDropNum, expectedDropNum); - String expected = (EXPECTED_SCHEMA + expectedNodes + EXPECTED_RELATIONSHIPS_PARAMS_ODD + expectedDrop) - .replace(NEO4J_SHELL.begin(), CYPHER_SHELL.begin()) - .replace(NEO4J_SHELL.commit(), CYPHER_SHELL.commit()) - .replace(NEO4J_SHELL.schemaAwait(), EXPECTED_INDEXES_AWAIT) - .replace(NEO4J_SHELL.schemaAwait(), CYPHER_SHELL.schemaAwait()); - assertEquals(expected, readFile(fileName)); + assertEquals(EXPECTED_QUERY_PARAMS_ODD, readFile(fileName)); } @Test @@ -1506,6 +1467,47 @@ static class ExportCypherResults { MATCH (n1:`UNIQUE IMPORT LABEL`{`UNIQUE IMPORT ID`:3}), (n2:`UNIQUE IMPORT LABEL`{`UNIQUE IMPORT ID`:4}) CREATE (n1)-[r:WORKS_FOR {id:1}]->(n2); MATCH (n1:`UNIQUE IMPORT LABEL`{`UNIQUE IMPORT ID`:5}), (n2:`UNIQUE IMPORT LABEL`{`UNIQUE IMPORT ID`:6}) CREATE (n1)-[r:WORKS_FOR {id:2}]->(n2); """; + + static final String EXPECTED_NODES_PARAMS_ODD = String.format(":begin%n" + + ":param rows => [{_id:4, properties:{age:12}}, {_id:5, properties:{age:4}}]%n" + + "UNWIND $rows AS row%n" + + "CREATE (n:`UNIQUE IMPORT LABEL`{`UNIQUE IMPORT ID`: row._id}) SET n += row.properties SET n:Bar;%n" + + ":param rows => [{_id:0, properties:{born:date('2018-10-31'), name:\"foo\"}}]%n" + + "UNWIND $rows AS row%n" + + "CREATE (n:`UNIQUE IMPORT LABEL`{`UNIQUE IMPORT ID`: row._id}) SET n += row.properties SET n:Foo;%n" + + ":commit%n" + + ":begin%n" + + ":param rows => [{_id:1, properties:{born:date('2017-09-29'), name:\"foo2\"}}, {_id:2, properties:{born:date('2016-03-12'), name:\"foo3\"}}]%n" + + "UNWIND $rows AS row%n" + + "CREATE (n:`UNIQUE IMPORT LABEL`{`UNIQUE IMPORT ID`: row._id}) SET n += row.properties SET n:Foo;%n" + + ":param rows => [{name:\"bar\", properties:{age:42}}]%n" + + "UNWIND $rows AS row%n" + + "CREATE (n:Bar{name: row.name}) SET n += row.properties;%n" + + ":commit%n" + + ":begin%n" + + ":param rows => [{name:\"bar2\", properties:{age:44}}, {name:\"bar3\", properties:{age:35}}]%n" + + "UNWIND $rows AS row%n" + + "CREATE (n:Bar{name: row.name}) SET n += row.properties;%n" + + ":param rows => [{name:\"bar4\", properties:{age:36}}]%n" + + "UNWIND $rows AS row%n" + + "CREATE (n:Bar{name: row.name}) SET n += row.properties;%n" + + ":commit%n"); + + static final String EXPECTED_DROP_PARAMS_ODD = String.format(":begin%n" + + "MATCH (n:`UNIQUE IMPORT LABEL`) WITH n LIMIT %1$d REMOVE n:`UNIQUE IMPORT LABEL` REMOVE n.`UNIQUE IMPORT ID`;%n" + + ":commit%n" + + ":begin%n" + + "MATCH (n:`UNIQUE IMPORT LABEL`) WITH n LIMIT %1$d REMOVE n:`UNIQUE IMPORT LABEL` REMOVE n.`UNIQUE IMPORT ID`;%n" + + ":commit%n" + + ":begin%n" + + "DROP CONSTRAINT UNIQUE_IMPORT_NAME;%n" + + ":commit%n", 3); + + static final String EXPECTED_QUERY_PARAMS_ODD = (EXPECTED_SCHEMA + EXPECTED_NODES_PARAMS_ODD + EXPECTED_RELATIONSHIPS_PARAMS_ODD + EXPECTED_DROP_PARAMS_ODD) + .replace(NEO4J_SHELL.begin(), CYPHER_SHELL.begin()) + .replace(NEO4J_SHELL.commit(), CYPHER_SHELL.commit()) + .replace(NEO4J_SHELL.schemaAwait(), EXPECTED_INDEXES_AWAIT) + .replace(NEO4J_SHELL.schemaAwait(), CYPHER_SHELL.schemaAwait()); private static String convertToCypherShellFormat(String input) { return input diff --git a/core/src/test/java/apoc/export/graphml/ExportGraphMLTestUtil.java b/core/src/test/java/apoc/export/graphml/ExportGraphMLTestUtil.java index 94e995cd6..f42c70208 100644 --- a/core/src/test/java/apoc/export/graphml/ExportGraphMLTestUtil.java +++ b/core/src/test/java/apoc/export/graphml/ExportGraphMLTestUtil.java @@ -178,9 +178,9 @@ public static void setUpGraphMl(GraphDatabaseService db, TestName testName) { TestUtil.registerProcedure(db, ExportGraphML.class, Graphs.class); apocConfig().setProperty(APOC_EXPORT_FILE_ENABLED, - Boolean.toString(!testName.getMethodName().endsWith("WithNoImportConfig"))); - apocConfig().setProperty(APOC_IMPORT_FILE_ENABLED, Boolean.toString(!testName.getMethodName().endsWith("WithNoExportConfig"))); + apocConfig().setProperty(APOC_IMPORT_FILE_ENABLED, + Boolean.toString(!testName.getMethodName().endsWith("WithNoImportConfig"))); apocConfig().setProperty(APOC_IMPORT_FILE_USE_NEO4J_CONFIG, false); db.executeTransactionally("CREATE (f:Foo:Foo2:Foo0 {name:'foo', born:Date('2018-10-10'), place:point({ longitude: 56.7, latitude: 12.78, height: 100 })})-[:KNOWS]->(b:Bar {name:'bar',age:42, place:point({ longitude: 56.7, latitude: 12.78})}),(c:Bar {age:12,values:[1,2,3]})"); diff --git a/core/src/test/java/apoc/export/json/ExportJsonS3Test.java b/core/src/test/java/apoc/export/json/ExportJsonS3Test.java index 5fce2824a..bc3f9dc1a 100644 --- a/core/src/test/java/apoc/export/json/ExportJsonS3Test.java +++ b/core/src/test/java/apoc/export/json/ExportJsonS3Test.java @@ -29,6 +29,7 @@ public class ExportJsonS3Test extends S3BaseTest { public void setUp() throws Exception { apocConfig().setProperty(APOC_EXPORT_FILE_ENABLED, true); TestUtil.registerProcedure(db, ExportJson.class, Graphs.class); + db.executeTransactionally("CREATE (f:User {name:'Adam',age:42,male:true,kids:['Sam','Anna','Grace'], born:localdatetime('2015185T19:32:24'), place:point({latitude: 13.1, longitude: 33.46789})})-[:KNOWS {since: 1993, bffSince: duration('P5M1.5D')}]->(b:User {name:'Jim',age:42}),(c:User {age:12})"); } @Test @@ -42,7 +43,7 @@ public void testExportAllJson() throws Exception { assertResults(s3Url, r, "database"); } ); - assertStreamStringEquals(directoryExpected, s3Url); + assertStreamStringEquals(directoryExpected, filename, s3Url); } @Test @@ -64,7 +65,7 @@ public void testExportPointMapDatetimeJson() throws Exception { assertEquals(s3Url, r.get("file")); assertEquals("json", r.get("format")); }); - assertStreamStringEquals(directoryExpected, s3Url); + assertStreamStringEquals(directoryExpected, filename, s3Url); } @Test @@ -80,7 +81,7 @@ public void testExportListNode() throws Exception { assertEquals(s3Url, r.get("file")); assertEquals("json", r.get("format")); }); - assertStreamStringEquals(directoryExpected, s3Url); + assertStreamStringEquals(directoryExpected, filename, s3Url); } @Test @@ -96,7 +97,7 @@ public void testExportListRel() throws Exception { assertEquals(s3Url, r.get("file")); assertEquals("json", r.get("format")); }); - assertStreamStringEquals(directoryExpected, s3Url); + assertStreamStringEquals(directoryExpected, filename, s3Url); } @Test @@ -112,7 +113,7 @@ public void testExportListPath() throws Exception { assertEquals(s3Url, r.get("file")); assertEquals("json", r.get("format")); }); - assertStreamStringEquals(directoryExpected, s3Url); + assertStreamStringEquals(directoryExpected, filename, s3Url); } @Test @@ -128,7 +129,7 @@ public void testExportMap() throws Exception { assertEquals(s3Url, r.get("file")); assertEquals("json", r.get("format")); }); - assertStreamStringEquals(directoryExpected, s3Url); + assertStreamStringEquals(directoryExpected, filename, s3Url); } @Test @@ -145,7 +146,7 @@ public void testExportMapPath() throws Exception { assertEquals(s3Url, r.get("file")); assertEquals("json", r.get("format")); }); - assertStreamStringEquals(directoryExpected, s3Url); + assertStreamStringEquals(directoryExpected, filename, s3Url); } @Test @@ -161,7 +162,7 @@ public void testExportMapRel() throws Exception { assertEquals(s3Url, r.get("file")); assertEquals("json", r.get("format")); }); - assertStreamStringEquals(directoryExpected, s3Url); + assertStreamStringEquals(directoryExpected, filename, s3Url); } @Test @@ -177,7 +178,7 @@ public void testExportMapComplex() throws Exception { assertEquals(s3Url, r.get("file")); assertEquals("json", r.get("format")); }); - assertStreamStringEquals(directoryExpected, s3Url); + assertStreamStringEquals(directoryExpected, filename, s3Url); } @Test @@ -191,7 +192,7 @@ public void testExportGraphJson() throws Exception { "RETURN *", map("s3", s3Url), (r) -> assertResults(s3Url, r, "graph")); - assertStreamStringEquals(directoryExpected, s3Url); + assertStreamStringEquals(directoryExpected, filename, s3Url); } @Test @@ -207,7 +208,7 @@ public void testExportQueryJson() throws Exception { assertEquals(s3Url, r.get("file")); assertEquals("json", r.get("format")); }); - assertStreamStringEquals(directoryExpected, s3Url); + assertStreamStringEquals(directoryExpected, filename, s3Url); } @Test @@ -223,7 +224,7 @@ public void testExportQueryNodesJson() throws Exception { assertEquals(s3Url, r.get("file")); assertEquals("json", r.get("format")); }); - assertStreamStringEquals(directoryExpected, s3Url); + assertStreamStringEquals(directoryExpected, filename, s3Url); } @Test @@ -239,7 +240,7 @@ public void testExportQueryTwoNodesJson() throws Exception { assertEquals(s3Url, r.get("file")); assertEquals("json", r.get("format")); }); - assertStreamStringEquals(directoryExpected, s3Url); + assertStreamStringEquals(directoryExpected, filename, s3Url); } @Test @@ -255,7 +256,7 @@ public void testExportQueryNodesJsonParams() throws Exception { assertEquals(s3Url, r.get("file")); assertEquals("json", r.get("format")); }); - assertStreamStringEquals(directoryExpected, s3Url); + assertStreamStringEquals(directoryExpected, filename, s3Url); } @Test @@ -271,7 +272,7 @@ public void testExportQueryNodesJsonCount() throws Exception { assertEquals(s3Url, r.get("file")); assertEquals("json", r.get("format")); }); - assertStreamStringEquals(directoryExpected, s3Url); + assertStreamStringEquals(directoryExpected, filename, s3Url); } @Test @@ -290,7 +291,7 @@ public void testExportData() throws Exception { assertEquals(s3Url, r.get("file")); assertEquals("json", r.get("format")); }); - assertStreamStringEquals(directoryExpected, s3Url); + assertStreamStringEquals(directoryExpected, filename, s3Url); } @Test @@ -305,7 +306,7 @@ public void testExportDataPath() throws Exception { assertEquals(s3Url, r.get("file")); assertEquals("json", r.get("format")); }); - assertStreamStringEquals(directoryExpected, s3Url); + assertStreamStringEquals(directoryExpected, filename, s3Url); } @Test @@ -321,7 +322,7 @@ public void testExportAllWithWriteNodePropertiesJson() throws Exception { assertEquals(s3Url, r.get("file")); assertEquals("json", r.get("format")); }); - assertStreamStringEquals(directoryExpected, s3Url); + assertStreamStringEquals(directoryExpected, filename, s3Url); } @Test @@ -338,7 +339,7 @@ public void testExportAllWithDefaultWriteNodePropertiesJson() throws Exception { assertEquals(s3Url, r.get("file")); assertEquals("json", r.get("format")); }); - assertStreamStringEquals(directoryExpected, s3Url); + assertStreamStringEquals(directoryExpected, filename, s3Url); } @Test @@ -354,7 +355,7 @@ public void testExportAllWithoutWriteNodePropertiesJson() throws Exception { assertEquals(s3Url, r.get("file")); assertEquals("json", r.get("format")); }); - assertStreamStringEquals(directoryExpected, s3Url); + assertStreamStringEquals(directoryExpected, filename, s3Url); } @Test @@ -371,12 +372,12 @@ public void testExportQueryOrderJson() throws Exception { assertEquals(s3Url, r.get("file")); assertEquals("json", r.get("format")); }); - assertStreamStringEquals(directoryExpected, s3Url); + assertStreamStringEquals(directoryExpected, filename, s3Url); } - private void assertStreamStringEquals(File directoryExpected, String s3Url) { + private void assertStreamStringEquals(File directoryExpected, String filename, String s3Url) { final String actual = readS3FileToString(s3Url); - assertStreamEquals(directoryExpected, s3Url, actual); + assertStreamEquals(directoryExpected, filename, actual); } private void assertResults(String filename, Map r, final String source) { diff --git a/core/src/test/java/apoc/export/json/ExportJsonTest.java b/core/src/test/java/apoc/export/json/ExportJsonTest.java index dbdb038b0..560a29138 100644 --- a/core/src/test/java/apoc/export/json/ExportJsonTest.java +++ b/core/src/test/java/apoc/export/json/ExportJsonTest.java @@ -564,6 +564,6 @@ private void assertStreamResults(Map r, final String source) { } private void assertStreamEquals(String fileName, String actualText) { - FileTestUtil.assertStreamEquals(directory, fileName, actualText); + FileTestUtil.assertStreamEquals(directoryExpected, fileName, actualText); } } From 8f75d31bf75a54e814bf253bc110837ccf8b2d16 Mon Sep 17 00:00:00 2001 From: Giuseppe Villani Date: Fri, 25 Nov 2022 11:00:37 +0100 Subject: [PATCH 3/3] lightened ExportS3PerformanceTest --- .../test/java/apoc/export/ExportS3PerformanceTest.java | 9 ++++----- test-utils/build.gradle | 2 +- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/core/src/test/java/apoc/export/ExportS3PerformanceTest.java b/core/src/test/java/apoc/export/ExportS3PerformanceTest.java index dbc77840e..8c370635c 100644 --- a/core/src/test/java/apoc/export/ExportS3PerformanceTest.java +++ b/core/src/test/java/apoc/export/ExportS3PerformanceTest.java @@ -3,6 +3,7 @@ import apoc.export.csv.ExportCSV; import apoc.graph.Graphs; import apoc.util.TestUtil; +import apoc.util.Util; import apoc.util.s3.S3BaseTest; import com.amazonaws.services.s3.model.S3Object; import org.junit.BeforeClass; @@ -14,6 +15,7 @@ import java.io.IOException; import java.time.Duration; import java.time.Instant; +import java.util.stream.IntStream; import static apoc.ApocConfig.APOC_EXPORT_FILE_ENABLED; import static apoc.ApocConfig.apocConfig; @@ -45,11 +47,8 @@ public static void setUp() throws Exception { @Test public void testExportAllCsvS3() throws Exception { System.out.println("Data creation started."); - // create large data (> 100 MB) - for (int i=0; i<555000; i++) { - String query = String.format("CREATE (f:User1:User {name:'foo%d',age:%d,male:true,kids:['a','b','c']})-[:KNOWS]->(b:User {name:'bar%d',age:%d}),(c:User {age:12})", i, i, i, i ); - db.executeTransactionally(query); - } + final String query = Util.readResourceFile("movies.cypher"); + IntStream.range(0, 5000).forEach(__-> db.executeTransactionally(query)); System.out.println("Data creation finished."); System.out.println("Test started."); diff --git a/test-utils/build.gradle b/test-utils/build.gradle index 9d424f97b..5aed7c253 100644 --- a/test-utils/build.gradle +++ b/test-utils/build.gradle @@ -44,7 +44,7 @@ dependencies { implementation group: 'org.neo4j', name: 'neo4j-io', version: neo4jVersionEffective, classifier: "tests" implementation group: 'org.gradle', name: 'gradle-tooling-api', version: '7.2' implementation group: 'org.jetbrains', name: 'annotations', version: "17.0.0" - implementation group: 'com.amazonaws', name: 'aws-java-sdk-s3', version: '1.11.270' + implementation group: 'com.amazonaws', name: 'aws-java-sdk-s3', version: '1.12.348' }