Skip to content

Commit

Permalink
Fixes neo4j-contrib#2890: The apoc.export.graphml.query export additi…
Browse files Browse the repository at this point in the history
…onal unwanted nodes
  • Loading branch information
vga91 committed Aug 2, 2022
1 parent fe9f8c7 commit 1e49557
Show file tree
Hide file tree
Showing 8 changed files with 158 additions and 19 deletions.
2 changes: 1 addition & 1 deletion core/src/main/java/apoc/export/cypher/ExportCypher.java
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ public Stream<DataProgressInfo> query(@Name("query") String query, @Name(value =
Result result = tx.execute(query);
SubGraph graph;
try {
graph = CypherResultSubGraph.from(tx, result, c.getRelsInBetween());
graph = CypherResultSubGraph.from(tx, result, c.getRelsInBetween(), c.isAddRelNodes());
} catch (IllegalStateException e) {
throw new RuntimeException("Full-text indexes on relationships are not supported, please delete them in order to complete the process");
}
Expand Down
2 changes: 1 addition & 1 deletion core/src/main/java/apoc/export/graphml/ExportGraphML.java
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ public Stream<ProgressInfo> graph(@Name("graph") Map<String,Object> graph, @Name
public Stream<ProgressInfo> query(@Name("query") String query, @Name("file") String fileName, @Name("config") Map<String, Object> config) throws Exception {
ExportConfig c = new ExportConfig(config);
Result result = tx.execute(query);
SubGraph graph = CypherResultSubGraph.from(tx, result, c.getRelsInBetween());
SubGraph graph = CypherResultSubGraph.from(tx, result, c.getRelsInBetween(), c.isAddRelNodes());
String source = String.format("statement: nodes(%d), rels(%d)",
Iterables.count(graph.getNodes()), Iterables.count(graph.getRelationships()));
return exportGraphML(fileName, source, graph, c);
Expand Down
7 changes: 6 additions & 1 deletion core/src/main/java/apoc/export/util/ExportConfig.java
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ public NodeConfig(Map<String, String> config) {
private Set<String> caption;
private boolean writeNodeProperties;
private boolean nodesOfRelationships;
private boolean addRelNodes;
private ExportFormat format;
private CypherFormat cypherFormat;
private final Map<String, Object> config;
Expand Down Expand Up @@ -133,6 +134,7 @@ public ExportConfig(Map<String,Object> config) {
this.samplingConfig = (Map<String, Object>) config.getOrDefault("samplingConfig", new HashMap<>());
this.unwindBatchSize = ((Number)getOptimizations().getOrDefault("unwindBatchSize", DEFAULT_UNWIND_BATCH_SIZE)).intValue();
this.awaitForIndexes = ((Number)config.getOrDefault("awaitForIndexes", 300)).longValue();
this.addRelNodes = toBoolean(config.getOrDefault("addRelNodes", true));
this.multipleRelationshipsWithType = toBoolean(config.get(RELS_WITH_TYPE_KEY));
this.source = new NodeConfig((Map<String, String>) config.get("source"));
this.target = new NodeConfig((Map<String, String>) config.get("target"));
Expand Down Expand Up @@ -161,7 +163,6 @@ private void exportQuotes(Map<String, Object> config)
this.quotes = toBoolean(config.get("quotes")) ? ALWAYS_QUOTES : NONE_QUOTES;
}
}

public boolean getRelsInBetween() {
return nodesOfRelationships;
}
Expand Down Expand Up @@ -250,6 +251,10 @@ public boolean ifNotExists() {
return ifNotExists;
}

public boolean isAddRelNodes() {
return addRelNodes;
}

public boolean shouldSaveIndexNames() {
return saveIndexNames;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,18 +37,24 @@ void addNode( long id, Node data )
labels.addAll( Iterables.asCollection( data.getLabels() ) );
}

public void add( Relationship rel )
public void add( Relationship rel, boolean addNodes )
{
final long id = rel.getId();
if ( !relationships.containsKey( id ) )
{
addRel( id, rel );
add( rel.getStartNode() );
add( rel.getEndNode() );
if (addNodes) {
add( rel.getStartNode() );
add( rel.getEndNode() );
}
}
}

public static SubGraph from(Transaction tx, Result result, boolean addBetween)
public static SubGraph from(Transaction tx, Result result, boolean addBetween) {
return from(tx, result, addBetween, true);
}

public static SubGraph from(Transaction tx, Result result, boolean addBetween, boolean addRelNodes)
{
final CypherResultSubGraph graph = new CypherResultSubGraph();
final List<String> columns = result.columns();
Expand All @@ -57,7 +63,7 @@ public static SubGraph from(Transaction tx, Result result, boolean addBetween)
for ( String column : columns )
{
final Object value = row.get( column );
graph.addToGraph( value );
graph.addToGraph( value, addRelNodes );
}
}
for ( IndexDefinition def : tx.schema().getIndexes() )
Expand Down Expand Up @@ -124,21 +130,21 @@ private void addRelationshipsBetweenNodes()
}
}

private void addToGraph( Object value )
private void addToGraph( Object value, boolean addNodes )
{
if ( value instanceof Node )
{
add( (Node) value );
}
if ( value instanceof Relationship )
{
add( (Relationship) value );
add( (Relationship) value, addNodes );
}
if ( value instanceof Iterable )
{
for ( Object inner : (Iterable) value )
{
addToGraph( inner );
addToGraph( inner, addNodes );
}
}
}
Expand Down
78 changes: 70 additions & 8 deletions core/src/test/java/apoc/export/cypher/ExportCypherTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,53 @@ public void testExportQueryCypherForNeo4j() throws Exception {
assertEquals(EXPECTED_NEO4J_SHELL, readFile(fileName));
}

@Test
public void testExportQueryOnlyRelCypherForNeo4j() throws Exception {
String fileName = "all.cypher";
String query = "MATCH (start)-[rel]->(end) RETURN rel";
final Map<String, Object> configMap = map("useOptimizations", map("type", "none"), "format", "neo4j-shell");
TestUtil.testCall(db, "CALL apoc.export.cypher.query($query,$file,$config)",
map("file", fileName, "query", query,
"config", configMap), (r) -> {});

final String expectedNodesWithAddRelNodes = String.format(EXPECTED_BEGIN_AND_FOO +
EXPECTED_BAR_END_NODE +
"COMMIT%n");
assertEquals(expectedNodesWithAddRelNodes + EXPECTED_SCHEMA + EXPECTED_RELATIONSHIPS + EXPECTED_CLEAN_UP, readFile(fileName));

// now the same config as above but with addRelNodes: false
configMap.put("addRelNodes", false);
TestUtil.testCall(db, "CALL apoc.export.cypher.query($query,$file,$config)",
map("file", fileName, "query", query,
"config", configMap), (r) -> {});

assertEquals(EXPECTED_REL_ONLY, readFile(fileName));
}

@Test
public void testExportQueryOnlyRelAndStartCypherForNeo4j() throws Exception {
String fileName = "all.cypher";
String query = "MATCH (start)-[rel]->(end) RETURN start, rel";
final Map<String, Object> configMap = map("useOptimizations", map("type", "none"), "format", "neo4j-shell", "addRelNodes", false);
TestUtil.testCall(db, "CALL apoc.export.cypher.query($query,$file,$config)",
map("file", fileName, "query", query,
"config", configMap), (r) -> {});

String expectedNodes = String.format(EXPECTED_BEGIN_AND_FOO + "COMMIT%n");
assertEquals(expectedNodes + EXPECTED_SCHEMA_ONLY_START + EXPECTED_REL_ONLY + EXPECTED_CLEAN_UP, readFile(fileName));

// now the same config as above but with nodesOfRelationships: true
configMap.put("nodesOfRelationships", true);
TestUtil.testCall(db, "CALL apoc.export.cypher.query($query,$file,$config)",
map("file", fileName, "query", query,
"config", configMap), (r) -> {});

String expectedNodesWithEndNode = String.format(EXPECTED_BEGIN_AND_FOO +
"CREATE (:Bar:`UNIQUE IMPORT LABEL` {age:42, name:\"bar\", `UNIQUE IMPORT ID`:1});%n" +
"COMMIT%n");
assertEquals(expectedNodesWithEndNode + EXPECTED_SCHEMA_ONLY_START + EXPECTED_REL_ONLY + EXPECTED_CLEAN_UP, readFile(fileName));
}

private static String readFile(String fileName) throws FileNotFoundException {
return TestUtil.readFileToString(new File(directory, fileName));
}
Expand Down Expand Up @@ -838,10 +885,13 @@ private void assertResultsOdd(String fileName, Map<String, Object> r) {

static class ExportCypherResults {

static final String EXPECTED_NODES = String.format("BEGIN%n" +
"CREATE (:Foo:`UNIQUE IMPORT LABEL` {born:date('2018-10-31'), name:\"foo\", `UNIQUE IMPORT ID`:0});%n" +
"CREATE (:Bar {age:42, name:\"bar\"});%n" +
"CREATE (:Bar:`UNIQUE IMPORT LABEL` {age:12, `UNIQUE IMPORT ID`:2});%n" +
static final String EXPECTED_ISOLATED_NODE = "CREATE (:Bar:`UNIQUE IMPORT LABEL` {age:12, `UNIQUE IMPORT ID`:2});\n";
static final String EXPECTED_BAR_END_NODE = "CREATE (:Bar {age:42, name:\"bar\"});%n";
static final String EXPECTED_BEGIN_AND_FOO = "BEGIN%n" +
"CREATE (:Foo:`UNIQUE IMPORT LABEL` {born:date('2018-10-31'), name:\"foo\", `UNIQUE IMPORT ID`:0});%n";
static final String EXPECTED_NODES = String.format(EXPECTED_BEGIN_AND_FOO +
EXPECTED_BAR_END_NODE +
EXPECTED_ISOLATED_NODE +
"COMMIT%n");

private static final String EXPECTED_NODES_MERGE = String.format("BEGIN%n" +
Expand All @@ -856,13 +906,25 @@ static class ExportCypherResults {
static final String EXPECTED_NODES_EMPTY = String.format("BEGIN%n" +
"COMMIT%n");

static final String EXPECTED_REL_ONLY = "BEGIN\n" +
"MATCH (n1:`UNIQUE IMPORT LABEL`{`UNIQUE IMPORT ID`:0}), (n2:`UNIQUE IMPORT LABEL`{`UNIQUE IMPORT ID`:1}) CREATE (n1)-[r:KNOWS {since:2016}]->(n2);\n" +
"COMMIT\n";


private static final String EXPECTED_CONSTRAINT_AND_AWAIT = "CREATE CONSTRAINT ON (node:`UNIQUE IMPORT LABEL`) ASSERT (node.`UNIQUE IMPORT ID`) IS UNIQUE;%n" +
"COMMIT%n" +
"SCHEMA AWAIT%n";

private static final String EXPECTED_IDX_FOO = "CREATE INDEX FOR (node:Foo) ON (node.name);%n";
static final String EXPECTED_SCHEMA = String.format("BEGIN%n" +
"CREATE INDEX FOR (node:Bar) ON (node.first_name, node.last_name);%n" +
"CREATE INDEX FOR (node:Foo) ON (node.name);%n" +
EXPECTED_IDX_FOO +
"CREATE CONSTRAINT ON (node:Bar) ASSERT (node.name) IS UNIQUE;%n" +
"CREATE CONSTRAINT ON (node:`UNIQUE IMPORT LABEL`) ASSERT (node.`UNIQUE IMPORT ID`) IS UNIQUE;%n" +
"COMMIT%n" +
"SCHEMA AWAIT%n");
EXPECTED_CONSTRAINT_AND_AWAIT);

static final String EXPECTED_SCHEMA_ONLY_START = String.format("BEGIN%n" +
EXPECTED_IDX_FOO +
EXPECTED_CONSTRAINT_AND_AWAIT);

static final String EXPECTED_SCHEMA_WITH_NAMES = "BEGIN%n" +
"CREATE INDEX%s FOR (node:Bar) ON (node.first_name, node.last_name);%n" +
Expand Down
60 changes: 60 additions & 0 deletions core/src/test/java/apoc/export/graphml/ExportGraphMLTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
import static apoc.util.BinaryTestUtil.fileToBinary;
import static apoc.util.MapUtil.map;
import static apoc.util.TestUtil.isRunningInCI;
import static apoc.util.TestUtil.testCall;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
Expand Down Expand Up @@ -175,6 +176,21 @@ public class ExportGraphMLTest {
private static final String EXPECTED_TYPES_EMPTY = String.format(HEADER + KEY_TYPES_EMPTY + GRAPH + DATA_EMPTY + FOOTER);
private static final String EXPECTED_TYPES_NO_DATA_KEY = String.format(HEADER + KEY_TYPES_NO_DATA_KEY + GRAPH + DATA_NO_DATA_KEY + FOOTER);

private static final String EDGES_QUERY = "<edge id=\"e1\" source=\"n3\" target=\"n4\" label=\"REL\"><data key=\"label\">REL</data><data key=\"foo\">bar</data></edge>%n";
private static final String EDGES_KEYS_QUERY = "<key id=\"foo\" for=\"edge\" attr.name=\"foo\"/>%n" +
"<key id=\"label\" for=\"edge\" attr.name=\"label\"/>%n";

private static final String START_NODE_QUERY = "<node id=\"n3\" labels=\":Start\"><data key=\"labels\">:Start</data><data key=\"startId\">1</data></node>%n";
private static final String START_NODE_KEYS_QUERY = "<key id=\"startId\" for=\"node\" attr.name=\"startId\"/>%n";
private static final String LABEL_KEY_QUERY = "<key id=\"labels\" for=\"node\" attr.name=\"labels\"/>%n";
private static final String END_NODE_QUERY = "<node id=\"n4\" labels=\":End\"><data key=\"labels\">:End</data><data key=\"endId\">1</data></node>%n";
private static final String END_NODE_KEYS_QUERY = "<key id=\"endId\" for=\"node\" attr.name=\"endId\"/>%n";
private static final String EXPECTED = String.format(HEADER +
END_NODE_KEYS_QUERY + START_NODE_KEYS_QUERY + LABEL_KEY_QUERY + EDGES_KEYS_QUERY +
GRAPH +
START_NODE_QUERY + END_NODE_QUERY + EDGES_QUERY +
FOOTER);

@Rule
public TestName testName = new TestName();

Expand Down Expand Up @@ -603,6 +619,7 @@ private void assertResultEmpty(File output, Map<String, Object> r) {
assertTrue("Should get time greater than 0", ((long) r.get("time")) > 0);
}

@Test
public void testExportGraphGraphMLQueryGephi() throws Exception {
File output = new File(directory, "query.graphml");
TestUtil.testCall(db, "call apoc.export.graphml.query('MATCH p=()-[r]->() RETURN p limit 1000',$file,{useTypes:true, format: 'gephi'}) ", map("file", output.getAbsolutePath()),
Expand All @@ -620,6 +637,49 @@ public void testExportGraphGraphMLQueryGephi() throws Exception {
});
assertXMLEquals(output, EXPECTED_TYPES_PATH);
}

@Test
public void testAddRelNodesWhenReturnOnlyRels() {
db.executeTransactionally("create (:Start {startId: 1})-[:REL {foo: 'bar'}]->(:End {endId: '1'})");

final String query = "CALL apoc.export.graphml.query('MATCH (start:Start)-[rel:REL]->(end:End) RETURN rel', null, $conf)\n" +
"YIELD data";

String expectedWithoutNodes = String.format(HEADER + EDGES_KEYS_QUERY + GRAPH + EDGES_QUERY + FOOTER);

testCall(db, query, map("conf", map( "stream", true)),
r -> assertXMLEquals(r.get("data"), EXPECTED));

testCall(db, query, map("conf", map("addRelNodes", false, "stream", true)),
r -> assertXMLEquals(r.get("data"), expectedWithoutNodes));

db.executeTransactionally("match (n) detach delete n");
}

@Test
public void testAddEndNodesOfRelationshipsWhenReturnOnlyStartNodeAndRel() {
db.executeTransactionally("create (:Start {startId: 1})-[:REL {foo: 'bar'}]->(:End {endId: '1'})");

String expectedWithoutEndNode = String.format(HEADER +
START_NODE_KEYS_QUERY + LABEL_KEY_QUERY + EDGES_KEYS_QUERY +
GRAPH +
START_NODE_QUERY + EDGES_QUERY +
FOOTER);

final String query2 = "CALL apoc.export.graphml.query('MATCH (start:Start)-[rel:REL]->(end:End) RETURN start, rel', null, $conf)\n" +
"YIELD data";

final Map<String, Object> confMap = map("stream", true, "addRelNodes", false);
testCall(db, query2, map("conf", confMap),
r -> assertXMLEquals(r.get("data"), expectedWithoutEndNode));

// now the same config as above but with nodesOfRelationships: true
confMap.put("nodesOfRelationships", true);
testCall(db, query2, map("conf", confMap),
r -> assertXMLEquals(r.get("data"), EXPECTED));

db.executeTransactionally("match (n) detach delete n");
}

@Test
public void testExportGraphGraphMLQueryGephiWithArrayCaption() throws Exception {
Expand Down
3 changes: 3 additions & 0 deletions docs/asciidoc/modules/ROOT/pages/export/graphml.adoc
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,9 @@ The procedures support the following config parameters:
| defaultRelationshipType | "RELATED" | set relationship type (import/export graphml)
| separateFiles | false | export results in separated file by type (nodes, relationships..)
| stream | false | stream the xml directly to the client into the `data` field
| addRelNodes | true | if enabled export start and end nodes, in case of query returning relationships
| nodesOfRelationships | false | if enabled export other terminal nodes, in case of query returning relationships and start or end nodes..
Note that both `addRelNodes` and `nodesOfRelationships` have to be false not to return other node, because with `addRelNodes=true` the start and end nodes are always returned.
| useTypes | false | Write the attribute type information to the graphml output
| source | Map<String,String> | Empty map | To be used together with `target` to import (via `apoc.import.graphml`) a relationships-only file. In this case the source and target attributes of `edge` tag are not based on an internal id of nodes but on a custom property value. +
For example, with a path like `(:Foo {name: "aaa"})-[:KNOWS]->(:Bar {age: 666})`, we can export the `KNOWS` rel with a config `<edge id="e2" source="aaa" sourceType="string" target="666" targetType="long" label="KNOWS"><data key="label">KNOWS</data><data key="id">1</data></edge>`. Note the additional `sourceType`/`targetType` to detect the right type during the import.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,4 +26,7 @@ The procedure support the following config parameters:
* `UNWIND_BATCH_PARAMS` - similar to `UNWIND_BATCH`, but also uses parameters where appropriate
| awaitForIndexes | Long | 300 | Timeout to use for `db.awaitIndexes` when using `format: "cypher-shell"`
| ifNotExists | boolean | false | If true adds the keyword `IF NOT EXISTS` to constraints and indexes
| addRelNodes | true | if enabled export start and end nodes, in case of query returning relationships
| nodesOfRelationships | false | if enabled export other terminal nodes, in case of query returning relationships and start or end nodes.
Note that both `addRelNodes` and `nodesOfRelationships` have to be false not to return other node, because with `addRelNodes=true` the start and end nodes are always returned.
|===

0 comments on commit 1e49557

Please sign in to comment.