export now creates 3 different files fixes neo4j-contrib#270 (neo4j-c…

…ontrib#374)

* Tests now comply with files separation fixes neo4j-contrib#270

* separateFiles config params fixes neo4j-contrib#270

src/main/java/apoc/export/cypher/ExportCypher.java

@@ -1,6 +1,5 @@
 package apoc.export.cypher;
 
-import org.neo4j.procedure.Description;
 import apoc.export.util.ExportConfig;
 import apoc.export.util.NodesAndRelsSubGraph;
 import apoc.export.util.ProgressReporter;
@@ -15,18 +14,17 @@
 import org.neo4j.graphdb.Result;
 import org.neo4j.helpers.collection.Iterables;
 import org.neo4j.procedure.Context;
+import org.neo4j.procedure.Description;
 import org.neo4j.procedure.Name;
 import org.neo4j.procedure.Procedure;
 
 import java.io.IOException;
-import java.io.PrintWriter;
 import java.util.Collection;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Stream;
 
 import static apoc.export.util.FileUtils.checkWriteAllowed;
-import static apoc.export.util.FileUtils.getPrintWriter;
 
 /**
  * @author mh
@@ -58,9 +56,10 @@ public Stream<ProgressInfo> data(@Name("nodes") List<Node> nodes, @Name("rels")
         String source = String.format("data: nodes(%d), rels(%d)", nodes.size(), rels.size());
         return exportCypher(fileName, source, new NodesAndRelsSubGraph(db, nodes, rels), new ExportConfig(config));
     }
+
     @Procedure
     @Description("apoc.export.cypher.graph(graph,file,config) - exports given graph object incl. indexes as cypher statements to the provided file")
-    public Stream<ProgressInfo> graph(@Name("graph") Map<String,Object> graph, @Name("file") String fileName, @Name("config") Map<String, Object> config) throws IOException {
+    public Stream<ProgressInfo> graph(@Name("graph") Map<String, Object> graph, @Name("file") String fileName, @Name("config") Map<String, Object> config) throws IOException {
 
         Collection<Node> nodes = (Collection<Node>) graph.get("nodes");
         Collection<Relationship> rels = (Collection<Relationship>) graph.get("relationships");
@@ -82,9 +81,9 @@ public Stream<ProgressInfo> query(@Name("query") String query, @Name("file") Str
     private Stream<ProgressInfo> exportCypher(@Name("file") String fileName, String source, SubGraph graph, ExportConfig c) throws IOException {
         checkWriteAllowed();
         ProgressReporter reporter = new ProgressReporter(null, null, new ProgressInfo(fileName, source, "cypher"));
-        PrintWriter printWriter = getPrintWriter(fileName, null);
         MultiStatementCypherSubGraphExporter exporter = new MultiStatementCypherSubGraphExporter(graph);
-        exporter.export(printWriter, c.getBatchSize(), reporter);
+        // Pass the full configuration to enable further enhancement
+        exporter.export(fileName, c, reporter);
         return reporter.stream();
     }
-}
+}

src/main/java/apoc/export/cypher/MultiStatementCypherSubGraphExporter.java

@@ -1,6 +1,8 @@
 package apoc.export.cypher;
 
 
+import apoc.export.util.ExportConfig;
+import apoc.export.util.FileUtils;
 import apoc.export.util.FormatUtils;
 import apoc.export.util.Reporter;
 import org.neo4j.cypher.export.SubGraph;
@@ -11,6 +13,7 @@
 import org.neo4j.graphdb.schema.IndexDefinition;
 import org.neo4j.helpers.collection.Iterables;
 
+import java.io.IOException;
 import java.io.PrintWriter;
 import java.lang.reflect.Array;
 import java.util.*;
@@ -19,42 +22,107 @@
  * Idea is to lookup nodes for relationships via a unqiue index
  * either one inherent to the original node, or a artificial one that indexes the original node-id
  * and which is removed after the import.
- *
+ * <p>
  * Outputs indexes and constraints at the beginning as their own transactions
  */
 public class MultiStatementCypherSubGraphExporter {
+    private final static String UNIQUE_ID_LABEL = "UNIQUE IMPORT LABEL";
+    private final static String Q_UNIQUE_ID_LABEL = quote(UNIQUE_ID_LABEL);
+    private final static String UNIQUE_ID_PROP = "UNIQUE IMPORT ID";
     private final SubGraph graph;
     private final Map<String, String> uniqueConstraints;
     Set<String> indexNames = new LinkedHashSet<>();
     Set<String> indexedProperties = new LinkedHashSet<>();
-    private final static String UNIQUE_ID_LABEL = "UNIQUE IMPORT LABEL";
-    private final static String Q_UNIQUE_ID_LABEL = quote(UNIQUE_ID_LABEL);
-    private final static String UNIQUE_ID_PROP = "UNIQUE IMPORT ID";
     private long artificialUniques = 0;
 
     public MultiStatementCypherSubGraphExporter(SubGraph graph) {
         this.graph = graph;
         uniqueConstraints = gatherUniqueConstraints(indexNames, indexedProperties);
     }
 
-    public void export(PrintWriter out, int batchSize, Reporter reporter) {
+    public static String quote(String id) {
+        return "`" + id + "`";
+    }
+
+    public static String label(String id) {
+        return ":`" + id + "`";
+    }
+
+    /**
+     * @param fileName
+     * @param suffix
+     * @return
+     */
+    private String normalizeFileName(final String fileName, String suffix) {
+        // TODO check if this should be follow the same rules of FileUtils.readerFor
+        return fileName.replace(".cypher", suffix != null ? "." + suffix + ".cypher" : ".cypher");
+    }
+
+    /**
+     * @param fileName
+     * @param suffix
+     * @return
+     * @throws IOException
+     */
+    private PrintWriter createWriter(String fileName, String suffix) throws IOException {
+        return FileUtils.getPrintWriter(normalizeFileName(fileName, suffix), null);
+    }
+
+    /**
+     * Given a full path file name like /tmp/myexport.cypher this method will create:
+     * <ul>
+     * <li>/tmp/myexport.schema.cypher</li>
+     * <li>/tmp/myexport.nodes.cypher</li>
+     * <li>/tmp/myexport.relationships.cypher</li>
+     * <li>/tmp/myexport.cleanup.cypher</li>
+     * </ul>
+     *
+     * @param fileName full path where all the files will be created
+     * @param config
+     * @param reporter
+     */
+    public void export(String fileName, ExportConfig config, Reporter reporter) throws IOException {
+        int batchSize = config.getBatchSize();
+
+        PrintWriter nodeWriter = null, schemaWriter = null, relationshipsWriter = null, cleanUpWriter = null;
+        if (config.separateFiles()) {
+            nodeWriter = createWriter(fileName, "nodes");
+            schemaWriter = createWriter(fileName, "schema");
+            relationshipsWriter = createWriter(fileName, "relationships");
+            cleanUpWriter = createWriter(fileName, "cleanup");
+        }
+        else {
+            // The same writer --> there is only one file
+            nodeWriter = createWriter(fileName, null);
+            schemaWriter = nodeWriter;
+            relationshipsWriter = nodeWriter;
+            cleanUpWriter = nodeWriter;
+        }
+
         boolean hasNodes = hasData(graph.getNodes());
         if (hasNodes) {
-            begin(out);
-            appendNodes(out, batchSize, reporter);
-            commit(out);
+            begin(nodeWriter);
+            appendNodes(nodeWriter, batchSize, reporter);
+            commit(nodeWriter);
+
+            nodeWriter.flush();
         }
-        writeMetaInformation(out);
+
+        writeMetaInformation(schemaWriter);
+        schemaWriter.flush();
 
         if (hasData(graph.getRelationships())) {
-            begin(out);
-            appendRelationships(out, batchSize, reporter);
-            commit(out);
+
+            begin(relationshipsWriter);
+            appendRelationships(relationshipsWriter, batchSize, reporter);
+            commit(relationshipsWriter);
+
+            relationshipsWriter.flush();
         }
         if (artificialUniques > 0) {
-            removeArtificialMetadata(out, batchSize);
+            removeArtificialMetadata(cleanUpWriter, batchSize);
         }
-        out.flush();
+        cleanUpWriter.flush();
     }
 
     private boolean hasData(Iterable<?> it) {
@@ -139,14 +207,6 @@ private String uniqueConstraint(String label, String key) {
         return "CREATE CONSTRAINT ON (node:" + quote(label) + ") ASSERT node." + quote(key) + " IS UNIQUE;";
     }
 
-    public static String quote(String id) {
-        return "`" + id + "`";
-    }
-
-    public static String label(String id) {
-        return ":`" + id + "`";
-    }
-
     private boolean hasProperties(PropertyContainer node) {
         return node.getPropertyKeys().iterator().hasNext();
     }
@@ -158,7 +218,8 @@ private String labelString(Node node) {
         while (labels.hasNext()) {
             Label next = labels.next();
             String labelName = next.name();
-            if (uniqueConstraints.containsKey(labelName) && node.hasProperty(uniqueConstraints.get(labelName))) uniqueFound = true;
+            if (uniqueConstraints.containsKey(labelName) && node.hasProperty(uniqueConstraints.get(labelName)))
+                uniqueFound = true;
             if (indexNames.contains(labelName))
                 result.insert(0, label(labelName));
             else
@@ -304,4 +365,4 @@ private String toString(Object value) {
         return value.toString();
     }
 
-}
+}

src/main/java/apoc/export/util/ExportConfig.java

@@ -85,4 +85,8 @@ public boolean readLabels() {
     public boolean storeNodeIds() {
         return toBoolean(config.getOrDefault("storeNodeIds",false));
     }
+
+    public boolean separateFiles(){
+        return toBoolean(config.getOrDefault("separateFiles", false));
+    }
 }

src/test/java/apoc/export/cypher/ExportCypherTest.java

@@ -1,6 +1,5 @@
 package apoc.export.cypher;
 
-import apoc.export.Export;
 import apoc.graph.Graphs;
 import apoc.util.TestUtil;
 import org.junit.AfterClass;
@@ -11,12 +10,12 @@
 import org.neo4j.test.TestGraphDatabaseFactory;
 
 import java.io.File;
+import java.util.Collections;
 import java.util.Map;
 import java.util.Scanner;
 
 import static apoc.util.MapUtil.map;
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
 
 /**
  * @author mh
@@ -45,6 +44,30 @@ public class ExportCypherTest {
             "DROP CONSTRAINT ON (node:`UNIQUE IMPORT LABEL`) ASSERT node.`UNIQUE IMPORT ID` IS UNIQUE;%n" +
             "commit");
 
+    private static final String EXPECTED_NODES = String.format("begin%n" +
+            "CREATE (:`Foo`:`UNIQUE IMPORT LABEL` {`name`:\"foo\", `UNIQUE IMPORT ID`:0});%n" +
+            "CREATE (:`Bar` {`name`:\"bar\", `age`:42});%n" +
+            "CREATE (:`Bar`:`UNIQUE IMPORT LABEL` {`age`:12, `UNIQUE IMPORT ID`:2});%n" +
+            "commit");
+
+    private static final String EXPECTED_SCHEMA = String.format("begin%n" +
+            "CREATE INDEX ON :`Foo`(`name`);%n" +
+            "CREATE CONSTRAINT ON (node:`Bar`) ASSERT node.`name` IS UNIQUE;%n" +
+            "CREATE CONSTRAINT ON (node:`UNIQUE IMPORT LABEL`) ASSERT node.`UNIQUE IMPORT ID` IS UNIQUE;%n" +
+            "commit%n" +
+            "schema await");
+
+    private static final String EXPECTED_RELATIONSHIPS = String.format("begin%n" +
+            "MATCH (n1:`UNIQUE IMPORT LABEL`{`UNIQUE IMPORT ID`:0}), (n2:`Bar`{`name`:\"bar\"}) CREATE (n1)-[:`KNOWS`]->(n2);%n" +
+            "commit");
+
+    private static final String EXPECTED_CLEAN_UP = String.format("begin%n" +
+            "MATCH (n:`UNIQUE IMPORT LABEL`)  WITH n LIMIT 20000 REMOVE n:`UNIQUE IMPORT LABEL` REMOVE n.`UNIQUE IMPORT ID`;%n" +
+            "commit%n" +
+            "begin%n" +
+            "DROP CONSTRAINT ON (node:`UNIQUE IMPORT LABEL`) ASSERT node.`UNIQUE IMPORT ID` IS UNIQUE;%n" +
+            "commit");
+private static final Map<String, Object> exportConfig = Collections.singletonMap("separateFiles", true);
     private static GraphDatabaseService db;
     private static File directory = new File("target/import");
 
@@ -61,7 +84,7 @@ public static void setUp() throws Exception {
                 .newGraphDatabase();
         TestUtil.registerProcedure(db, ExportCypher.class, Graphs.class);
         db.execute("CREATE INDEX ON :Foo(name)").close();
-        db.execute("CREATE CONSTRAINT ON (b:Bar) ASSERT b.name is unique").close();
+        db.execute("CREATE CONSTRAINT ON (b:Bar) ASSERT b.name IS UNIQUE").close();
         db.execute("CREATE (f:Foo {name:'foo'})-[:KNOWS]->(b:Bar {name:'bar',age:42}),(c:Bar {age:12})").close();
     }
 
@@ -70,6 +93,7 @@ public static void tearDown() {
         db.shutdown();
     }
 
+    // -- Whole file test -- //
     @Test
     public void testExportAllCypher() throws Exception {
         File output = new File(directory, "all.cypher");
@@ -89,6 +113,83 @@ public void testExportGraphCypher() throws Exception {
         assertEquals(EXPECTED, new Scanner(output).useDelimiter("\\Z").next());
     }
 
+    // -- Separate files tests -- //
+    @Test
+    public void testExportAllCypherNodes() throws Exception {
+        File output = new File(directory, "all.cypher");
+        TestUtil.testCall(db, "CALL apoc.export.cypher.all({file},{exportConfig})", map("file", output.getAbsolutePath(), "exportConfig", exportConfig),
+                (r) -> assertResults(output, r, "database"));
+        assertEquals(EXPECTED_NODES, new Scanner(new File(directory, "all.nodes.cypher")).useDelimiter("\\Z").next());
+    }
+
+    @Test
+    public void testExportAllCypherRelationships() throws Exception {
+        File output = new File(directory, "all.cypher");
+        TestUtil.testCall(db, "CALL apoc.export.cypher.all({file},{exportConfig})", map("file", output.getAbsolutePath(), "exportConfig", exportConfig),
+                (r) -> assertResults(output, r, "database"));
+        assertEquals(EXPECTED_RELATIONSHIPS, new Scanner(new File(directory, "all.relationships.cypher")).useDelimiter("\\Z").next());
+    }
+
+    @Test
+    public void testExportAllCypherSchema() throws Exception {
+        File output = new File(directory, "all.cypher");
+        TestUtil.testCall(db, "CALL apoc.export.cypher.all({file},{exportConfig})", map("file", output.getAbsolutePath(), "exportConfig", exportConfig),
+                (r) -> assertResults(output, r, "database"));
+        assertEquals(EXPECTED_SCHEMA, new Scanner(new File(directory, "all.schema.cypher")).useDelimiter("\\Z").next());
+    }
+
+    @Test
+    public void testExportAllCypherCleanUp() throws Exception {
+        File output = new File(directory, "all.cypher");
+        TestUtil.testCall(db, "CALL apoc.export.cypher.all({file},{exportConfig})", map("file", output.getAbsolutePath(), "exportConfig", exportConfig),
+                (r) -> assertResults(output, r, "database"));
+        assertEquals(EXPECTED_CLEAN_UP, new Scanner(new File(directory, "all.cleanup.cypher")).useDelimiter("\\Z").next());
+    }
+
+    @Test
+    public void testExportGraphCypherNodes() throws Exception {
+        File output = new File(directory, "graph.cypher");
+        TestUtil.testCall(db, "CALL apoc.graph.fromDB('test',{}) yield graph " +
+                        "CALL apoc.export.cypher.graph(graph, {file},{exportConfig}) " +
+                        "YIELD nodes, relationships, properties, file, source,format, time " +
+                        "RETURN *", map("file", output.getAbsolutePath(), "exportConfig", exportConfig),
+                (r) -> assertResults(output, r, "graph"));
+        assertEquals(EXPECTED_NODES, new Scanner(new File(directory, "graph.nodes.cypher")).useDelimiter("\\Z").next());
+    }
+
+    @Test
+    public void testExportGraphCypherRelationships() throws Exception {
+        File output = new File(directory, "graph.cypher");
+        TestUtil.testCall(db, "CALL apoc.graph.fromDB('test',{}) yield graph " +
+                        "CALL apoc.export.cypher.graph(graph, {file},{exportConfig}) " +
+                        "YIELD nodes, relationships, properties, file, source,format, time " +
+                        "RETURN *", map("file", output.getAbsolutePath(), "exportConfig", exportConfig),
+                (r) -> assertResults(output, r, "graph"));
+        assertEquals(EXPECTED_RELATIONSHIPS, new Scanner(new File(directory, "graph.relationships.cypher")).useDelimiter("\\Z").next());
+    }
+
+    @Test
+    public void testExportGraphCypherSchema() throws Exception {
+        File output = new File(directory, "graph.cypher");
+        TestUtil.testCall(db, "CALL apoc.graph.fromDB('test',{}) yield graph " +
+                        "CALL apoc.export.cypher.graph(graph, {file},{exportConfig}) " +
+                        "YIELD nodes, relationships, properties, file, source,format, time " +
+                        "RETURN *", map("file", output.getAbsolutePath(), "exportConfig", exportConfig),
+                (r) -> assertResults(output, r, "graph"));
+        assertEquals(EXPECTED_SCHEMA, new Scanner(new File(directory, "graph.schema.cypher")).useDelimiter("\\Z").next());
+    }
+
+    @Test
+    public void testExportGraphCypherCleanUp() throws Exception {
+        File output = new File(directory, "graph.cypher");
+        TestUtil.testCall(db, "CALL apoc.graph.fromDB('test',{}) yield graph " +
+                        "CALL apoc.export.cypher.graph(graph, {file},{exportConfig}) " +
+                        "YIELD nodes, relationships, properties, file, source,format, time " +
+                        "RETURN *", map("file", output.getAbsolutePath(), "exportConfig", exportConfig),
+                (r) -> assertResults(output, r, "graph"));
+        assertEquals(EXPECTED_CLEAN_UP, new Scanner(new File(directory, "graph.cleanup.cypher")).useDelimiter("\\Z").next());
+    }
+
     private void assertResults(File output, Map<String, Object> r, final String source) {
         assertEquals(3L, r.get("nodes"));
         assertEquals(1L, r.get("relationships"));