diff --git a/src/main/java/apoc/export/cypher/ExportCypher.java b/src/main/java/apoc/export/cypher/ExportCypher.java index 346ea91093..f96332a6e9 100644 --- a/src/main/java/apoc/export/cypher/ExportCypher.java +++ b/src/main/java/apoc/export/cypher/ExportCypher.java @@ -1,6 +1,5 @@ package apoc.export.cypher; -import org.neo4j.procedure.Description; import apoc.export.util.ExportConfig; import apoc.export.util.NodesAndRelsSubGraph; import apoc.export.util.ProgressReporter; @@ -15,18 +14,17 @@ import org.neo4j.graphdb.Result; import org.neo4j.helpers.collection.Iterables; import org.neo4j.procedure.Context; +import org.neo4j.procedure.Description; import org.neo4j.procedure.Name; import org.neo4j.procedure.Procedure; import java.io.IOException; -import java.io.PrintWriter; import java.util.Collection; import java.util.List; import java.util.Map; import java.util.stream.Stream; import static apoc.export.util.FileUtils.checkWriteAllowed; -import static apoc.export.util.FileUtils.getPrintWriter; /** * @author mh @@ -58,9 +56,10 @@ public Stream data(@Name("nodes") List nodes, @Name("rels") String source = String.format("data: nodes(%d), rels(%d)", nodes.size(), rels.size()); return exportCypher(fileName, source, new NodesAndRelsSubGraph(db, nodes, rels), new ExportConfig(config)); } + @Procedure @Description("apoc.export.cypher.graph(graph,file,config) - exports given graph object incl. indexes as cypher statements to the provided file") - public Stream graph(@Name("graph") Map graph, @Name("file") String fileName, @Name("config") Map config) throws IOException { + public Stream graph(@Name("graph") Map graph, @Name("file") String fileName, @Name("config") Map config) throws IOException { Collection nodes = (Collection) graph.get("nodes"); Collection rels = (Collection) graph.get("relationships"); @@ -82,9 +81,9 @@ public Stream query(@Name("query") String query, @Name("file") Str private Stream exportCypher(@Name("file") String fileName, String source, SubGraph graph, ExportConfig c) throws IOException { checkWriteAllowed(); ProgressReporter reporter = new ProgressReporter(null, null, new ProgressInfo(fileName, source, "cypher")); - PrintWriter printWriter = getPrintWriter(fileName, null); MultiStatementCypherSubGraphExporter exporter = new MultiStatementCypherSubGraphExporter(graph); - exporter.export(printWriter, c.getBatchSize(), reporter); + // Pass the full configuration to enable further enhancement + exporter.export(fileName, c, reporter); return reporter.stream(); } -} +} \ No newline at end of file diff --git a/src/main/java/apoc/export/cypher/MultiStatementCypherSubGraphExporter.java b/src/main/java/apoc/export/cypher/MultiStatementCypherSubGraphExporter.java index a77f221b83..af9d85350c 100644 --- a/src/main/java/apoc/export/cypher/MultiStatementCypherSubGraphExporter.java +++ b/src/main/java/apoc/export/cypher/MultiStatementCypherSubGraphExporter.java @@ -1,6 +1,8 @@ package apoc.export.cypher; +import apoc.export.util.ExportConfig; +import apoc.export.util.FileUtils; import apoc.export.util.FormatUtils; import apoc.export.util.Reporter; import org.neo4j.cypher.export.SubGraph; @@ -11,6 +13,7 @@ import org.neo4j.graphdb.schema.IndexDefinition; import org.neo4j.helpers.collection.Iterables; +import java.io.IOException; import java.io.PrintWriter; import java.lang.reflect.Array; import java.util.*; @@ -19,17 +22,17 @@ * Idea is to lookup nodes for relationships via a unqiue index * either one inherent to the original node, or a artificial one that indexes the original node-id * and which is removed after the import. - * + *

* Outputs indexes and constraints at the beginning as their own transactions */ public class MultiStatementCypherSubGraphExporter { + private final static String UNIQUE_ID_LABEL = "UNIQUE IMPORT LABEL"; + private final static String Q_UNIQUE_ID_LABEL = quote(UNIQUE_ID_LABEL); + private final static String UNIQUE_ID_PROP = "UNIQUE IMPORT ID"; private final SubGraph graph; private final Map uniqueConstraints; Set indexNames = new LinkedHashSet<>(); Set indexedProperties = new LinkedHashSet<>(); - private final static String UNIQUE_ID_LABEL = "UNIQUE IMPORT LABEL"; - private final static String Q_UNIQUE_ID_LABEL = quote(UNIQUE_ID_LABEL); - private final static String UNIQUE_ID_PROP = "UNIQUE IMPORT ID"; private long artificialUniques = 0; public MultiStatementCypherSubGraphExporter(SubGraph graph) { @@ -37,24 +40,89 @@ public MultiStatementCypherSubGraphExporter(SubGraph graph) { uniqueConstraints = gatherUniqueConstraints(indexNames, indexedProperties); } - public void export(PrintWriter out, int batchSize, Reporter reporter) { + public static String quote(String id) { + return "`" + id + "`"; + } + + public static String label(String id) { + return ":`" + id + "`"; + } + + /** + * @param fileName + * @param suffix + * @return + */ + private String normalizeFileName(final String fileName, String suffix) { + // TODO check if this should be follow the same rules of FileUtils.readerFor + return fileName.replace(".cypher", suffix != null ? "." + suffix + ".cypher" : ".cypher"); + } + + /** + * @param fileName + * @param suffix + * @return + * @throws IOException + */ + private PrintWriter createWriter(String fileName, String suffix) throws IOException { + return FileUtils.getPrintWriter(normalizeFileName(fileName, suffix), null); + } + + /** + * Given a full path file name like /tmp/myexport.cypher this method will create: + *

    + *
  • /tmp/myexport.schema.cypher
    • + *
  • /tmp/myexport.nodes.cypher
    • + *
  • /tmp/myexport.relationships.cypher
    • + *
  • /tmp/myexport.cleanup.cypher
    • + *
+ * + * @param fileName full path where all the files will be created + * @param config + * @param reporter + */ + public void export(String fileName, ExportConfig config, Reporter reporter) throws IOException { + int batchSize = config.getBatchSize(); + + PrintWriter nodeWriter = null, schemaWriter = null, relationshipsWriter = null, cleanUpWriter = null; + if (config.separateFiles()) { + nodeWriter = createWriter(fileName, "nodes"); + schemaWriter = createWriter(fileName, "schema"); + relationshipsWriter = createWriter(fileName, "relationships"); + cleanUpWriter = createWriter(fileName, "cleanup"); + } + else { + // The same writer --> there is only one file + nodeWriter = createWriter(fileName, null); + schemaWriter = nodeWriter; + relationshipsWriter = nodeWriter; + cleanUpWriter = nodeWriter; + } + boolean hasNodes = hasData(graph.getNodes()); if (hasNodes) { - begin(out); - appendNodes(out, batchSize, reporter); - commit(out); + begin(nodeWriter); + appendNodes(nodeWriter, batchSize, reporter); + commit(nodeWriter); + + nodeWriter.flush(); } - writeMetaInformation(out); + + writeMetaInformation(schemaWriter); + schemaWriter.flush(); if (hasData(graph.getRelationships())) { - begin(out); - appendRelationships(out, batchSize, reporter); - commit(out); + + begin(relationshipsWriter); + appendRelationships(relationshipsWriter, batchSize, reporter); + commit(relationshipsWriter); + + relationshipsWriter.flush(); } if (artificialUniques > 0) { - removeArtificialMetadata(out, batchSize); + removeArtificialMetadata(cleanUpWriter, batchSize); } - out.flush(); + cleanUpWriter.flush(); } private boolean hasData(Iterable it) { @@ -139,14 +207,6 @@ private String uniqueConstraint(String label, String key) { return "CREATE CONSTRAINT ON (node:" + quote(label) + ") ASSERT node." + quote(key) + " IS UNIQUE;"; } - public static String quote(String id) { - return "`" + id + "`"; - } - - public static String label(String id) { - return ":`" + id + "`"; - } - private boolean hasProperties(PropertyContainer node) { return node.getPropertyKeys().iterator().hasNext(); } @@ -158,7 +218,8 @@ private String labelString(Node node) { while (labels.hasNext()) { Label next = labels.next(); String labelName = next.name(); - if (uniqueConstraints.containsKey(labelName) && node.hasProperty(uniqueConstraints.get(labelName))) uniqueFound = true; + if (uniqueConstraints.containsKey(labelName) && node.hasProperty(uniqueConstraints.get(labelName))) + uniqueFound = true; if (indexNames.contains(labelName)) result.insert(0, label(labelName)); else @@ -304,4 +365,4 @@ private String toString(Object value) { return value.toString(); } -} +} \ No newline at end of file diff --git a/src/main/java/apoc/export/util/ExportConfig.java b/src/main/java/apoc/export/util/ExportConfig.java index f6e65188ac..89ec6bf7dc 100644 --- a/src/main/java/apoc/export/util/ExportConfig.java +++ b/src/main/java/apoc/export/util/ExportConfig.java @@ -85,4 +85,8 @@ public boolean readLabels() { public boolean storeNodeIds() { return toBoolean(config.getOrDefault("storeNodeIds",false)); } + + public boolean separateFiles(){ + return toBoolean(config.getOrDefault("separateFiles", false)); + } } diff --git a/src/test/java/apoc/export/cypher/ExportCypherTest.java b/src/test/java/apoc/export/cypher/ExportCypherTest.java index fe301d1eda..373b1fd015 100644 --- a/src/test/java/apoc/export/cypher/ExportCypherTest.java +++ b/src/test/java/apoc/export/cypher/ExportCypherTest.java @@ -1,6 +1,5 @@ package apoc.export.cypher; -import apoc.export.Export; import apoc.graph.Graphs; import apoc.util.TestUtil; import org.junit.AfterClass; @@ -11,12 +10,12 @@ import org.neo4j.test.TestGraphDatabaseFactory; import java.io.File; +import java.util.Collections; import java.util.Map; import java.util.Scanner; import static apoc.util.MapUtil.map; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; /** * @author mh @@ -45,6 +44,30 @@ public class ExportCypherTest { "DROP CONSTRAINT ON (node:`UNIQUE IMPORT LABEL`) ASSERT node.`UNIQUE IMPORT ID` IS UNIQUE;%n" + "commit"); + private static final String EXPECTED_NODES = String.format("begin%n" + + "CREATE (:`Foo`:`UNIQUE IMPORT LABEL` {`name`:\"foo\", `UNIQUE IMPORT ID`:0});%n" + + "CREATE (:`Bar` {`name`:\"bar\", `age`:42});%n" + + "CREATE (:`Bar`:`UNIQUE IMPORT LABEL` {`age`:12, `UNIQUE IMPORT ID`:2});%n" + + "commit"); + + private static final String EXPECTED_SCHEMA = String.format("begin%n" + + "CREATE INDEX ON :`Foo`(`name`);%n" + + "CREATE CONSTRAINT ON (node:`Bar`) ASSERT node.`name` IS UNIQUE;%n" + + "CREATE CONSTRAINT ON (node:`UNIQUE IMPORT LABEL`) ASSERT node.`UNIQUE IMPORT ID` IS UNIQUE;%n" + + "commit%n" + + "schema await"); + + private static final String EXPECTED_RELATIONSHIPS = String.format("begin%n" + + "MATCH (n1:`UNIQUE IMPORT LABEL`{`UNIQUE IMPORT ID`:0}), (n2:`Bar`{`name`:\"bar\"}) CREATE (n1)-[:`KNOWS`]->(n2);%n" + + "commit"); + + private static final String EXPECTED_CLEAN_UP = String.format("begin%n" + + "MATCH (n:`UNIQUE IMPORT LABEL`) WITH n LIMIT 20000 REMOVE n:`UNIQUE IMPORT LABEL` REMOVE n.`UNIQUE IMPORT ID`;%n" + + "commit%n" + + "begin%n" + + "DROP CONSTRAINT ON (node:`UNIQUE IMPORT LABEL`) ASSERT node.`UNIQUE IMPORT ID` IS UNIQUE;%n" + + "commit"); +private static final Map exportConfig = Collections.singletonMap("separateFiles", true); private static GraphDatabaseService db; private static File directory = new File("target/import"); @@ -61,7 +84,7 @@ public static void setUp() throws Exception { .newGraphDatabase(); TestUtil.registerProcedure(db, ExportCypher.class, Graphs.class); db.execute("CREATE INDEX ON :Foo(name)").close(); - db.execute("CREATE CONSTRAINT ON (b:Bar) ASSERT b.name is unique").close(); + db.execute("CREATE CONSTRAINT ON (b:Bar) ASSERT b.name IS UNIQUE").close(); db.execute("CREATE (f:Foo {name:'foo'})-[:KNOWS]->(b:Bar {name:'bar',age:42}),(c:Bar {age:12})").close(); } @@ -70,6 +93,7 @@ public static void tearDown() { db.shutdown(); } + // -- Whole file test -- // @Test public void testExportAllCypher() throws Exception { File output = new File(directory, "all.cypher"); @@ -89,6 +113,83 @@ public void testExportGraphCypher() throws Exception { assertEquals(EXPECTED, new Scanner(output).useDelimiter("\\Z").next()); } + // -- Separate files tests -- // + @Test + public void testExportAllCypherNodes() throws Exception { + File output = new File(directory, "all.cypher"); + TestUtil.testCall(db, "CALL apoc.export.cypher.all({file},{exportConfig})", map("file", output.getAbsolutePath(), "exportConfig", exportConfig), + (r) -> assertResults(output, r, "database")); + assertEquals(EXPECTED_NODES, new Scanner(new File(directory, "all.nodes.cypher")).useDelimiter("\\Z").next()); + } + + @Test + public void testExportAllCypherRelationships() throws Exception { + File output = new File(directory, "all.cypher"); + TestUtil.testCall(db, "CALL apoc.export.cypher.all({file},{exportConfig})", map("file", output.getAbsolutePath(), "exportConfig", exportConfig), + (r) -> assertResults(output, r, "database")); + assertEquals(EXPECTED_RELATIONSHIPS, new Scanner(new File(directory, "all.relationships.cypher")).useDelimiter("\\Z").next()); + } + + @Test + public void testExportAllCypherSchema() throws Exception { + File output = new File(directory, "all.cypher"); + TestUtil.testCall(db, "CALL apoc.export.cypher.all({file},{exportConfig})", map("file", output.getAbsolutePath(), "exportConfig", exportConfig), + (r) -> assertResults(output, r, "database")); + assertEquals(EXPECTED_SCHEMA, new Scanner(new File(directory, "all.schema.cypher")).useDelimiter("\\Z").next()); + } + + @Test + public void testExportAllCypherCleanUp() throws Exception { + File output = new File(directory, "all.cypher"); + TestUtil.testCall(db, "CALL apoc.export.cypher.all({file},{exportConfig})", map("file", output.getAbsolutePath(), "exportConfig", exportConfig), + (r) -> assertResults(output, r, "database")); + assertEquals(EXPECTED_CLEAN_UP, new Scanner(new File(directory, "all.cleanup.cypher")).useDelimiter("\\Z").next()); + } + + @Test + public void testExportGraphCypherNodes() throws Exception { + File output = new File(directory, "graph.cypher"); + TestUtil.testCall(db, "CALL apoc.graph.fromDB('test',{}) yield graph " + + "CALL apoc.export.cypher.graph(graph, {file},{exportConfig}) " + + "YIELD nodes, relationships, properties, file, source,format, time " + + "RETURN *", map("file", output.getAbsolutePath(), "exportConfig", exportConfig), + (r) -> assertResults(output, r, "graph")); + assertEquals(EXPECTED_NODES, new Scanner(new File(directory, "graph.nodes.cypher")).useDelimiter("\\Z").next()); + } + + @Test + public void testExportGraphCypherRelationships() throws Exception { + File output = new File(directory, "graph.cypher"); + TestUtil.testCall(db, "CALL apoc.graph.fromDB('test',{}) yield graph " + + "CALL apoc.export.cypher.graph(graph, {file},{exportConfig}) " + + "YIELD nodes, relationships, properties, file, source,format, time " + + "RETURN *", map("file", output.getAbsolutePath(), "exportConfig", exportConfig), + (r) -> assertResults(output, r, "graph")); + assertEquals(EXPECTED_RELATIONSHIPS, new Scanner(new File(directory, "graph.relationships.cypher")).useDelimiter("\\Z").next()); + } + + @Test + public void testExportGraphCypherSchema() throws Exception { + File output = new File(directory, "graph.cypher"); + TestUtil.testCall(db, "CALL apoc.graph.fromDB('test',{}) yield graph " + + "CALL apoc.export.cypher.graph(graph, {file},{exportConfig}) " + + "YIELD nodes, relationships, properties, file, source,format, time " + + "RETURN *", map("file", output.getAbsolutePath(), "exportConfig", exportConfig), + (r) -> assertResults(output, r, "graph")); + assertEquals(EXPECTED_SCHEMA, new Scanner(new File(directory, "graph.schema.cypher")).useDelimiter("\\Z").next()); + } + + @Test + public void testExportGraphCypherCleanUp() throws Exception { + File output = new File(directory, "graph.cypher"); + TestUtil.testCall(db, "CALL apoc.graph.fromDB('test',{}) yield graph " + + "CALL apoc.export.cypher.graph(graph, {file},{exportConfig}) " + + "YIELD nodes, relationships, properties, file, source,format, time " + + "RETURN *", map("file", output.getAbsolutePath(), "exportConfig", exportConfig), + (r) -> assertResults(output, r, "graph")); + assertEquals(EXPECTED_CLEAN_UP, new Scanner(new File(directory, "graph.cleanup.cypher")).useDelimiter("\\Z").next()); + } + private void assertResults(File output, Map r, final String source) { assertEquals(3L, r.get("nodes")); assertEquals(1L, r.get("relationships"));