From c49fe8a2ea99001de9d2b3640ec813708aae1962 Mon Sep 17 00:00:00 2001 From: Giuseppe Villani Date: Thu, 11 Jul 2024 16:48:12 +0200 Subject: [PATCH] Solve dev 5.21 build errors (#4140) * Try to solve 5.21 TC errors (#4137) * Try to solve 5.21 TC errors * de-ignored gradle.properties to be consistent with Core one * Solve 5.21 compile errors (#4126) * Solve 5.21 compile errors * Added CypherScope.CYPHER_5 * Fix compile 5.21 errors after restore submodule * solve error: cannot find symbol, due to org.neo4j.kernel.api.procedure.ProcedureView 5.21 changes * fix ParquetTest.testStreamRoundtripParquetAllWithImportExportConfsDisabled * fix LoadCsvTest.testWithSpacesInFileName * fix SystemDbTest.testGetGraph * Update .gitmodules to 5.21 --- .gitignore | 1 - .gitmodules | 2 +- .../test/java/apoc/vectordb/ChromaDbTest.java | 4 ++-- .../test/java/apoc/vectordb/QdrantTest.java | 2 +- .../apoc/custom/CypherProceduresHandler.java | 15 +++---------- .../apoc/custom/CypherProceduresUtil.java | 10 +++++---- .../src/main/java/apoc/custom/Signatures.java | 6 ++--- .../main/java/apoc/cypher/CypherExtended.java | 5 ++--- .../java/apoc/export/arrow/ImportArrow.java | 4 +++- .../apoc/export/parquet/ExportParquet.java | 17 +++++++++----- .../apoc/export/parquet/ParquetReadUtil.java | 6 +++-- extended/src/main/java/apoc/load/LoadCsv.java | 2 +- .../main/java/apoc/load/LoadDirectory.java | 13 ++++++----- .../src/main/java/apoc/load/LoadHtml.java | 5 ++++- .../java/apoc/export/parquet/ParquetTest.java | 2 +- .../src/test/java/apoc/load/LoadCsvTest.java | 2 +- .../test/java/apoc/systemdb/SystemDbTest.java | 6 +++-- .../test/java/apoc/util/ExtendedTestUtil.java | 8 +++++++ .../test/java/apoc/util/FileUtilsTest.java | 22 ++++++++++--------- gradle.properties | 1 + 20 files changed, 75 insertions(+), 58 deletions(-) create mode 100644 gradle.properties diff --git a/.gitignore b/.gitignore index f308466fcf..6ebb99b4a7 100644 --- a/.gitignore +++ b/.gitignore @@ -6,7 +6,6 @@ classes dependency-reduced-pom.xml derby* .gradle -gradle.properties build/ *~ \#* diff --git a/.gitmodules b/.gitmodules index b051f40471..a485ec58b0 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +1,4 @@ [submodule "apoc-core"] path = apoc-core url = https://github.com/neo4j/apoc - branch = 5.20 + branch = 5.21 diff --git a/extended-it/src/test/java/apoc/vectordb/ChromaDbTest.java b/extended-it/src/test/java/apoc/vectordb/ChromaDbTest.java index 3e90b52883..291caf8424 100644 --- a/extended-it/src/test/java/apoc/vectordb/ChromaDbTest.java +++ b/extended-it/src/test/java/apoc/vectordb/ChromaDbTest.java @@ -69,7 +69,7 @@ public static void setUp() throws Exception { CHROMA_CONTAINER.start(); - HOST = "localhost:" + CHROMA_CONTAINER.getMappedPort(8000); + HOST = CHROMA_CONTAINER.getEndpoint(); TestUtil.registerProcedure(db, ChromaDb.class, VectorDb.class, Prompt.class); testCall(db, "CALL apoc.vectordb.chroma.createCollection($host, 'test_collection', 'cosine', 4)", @@ -396,7 +396,7 @@ MAPPING_KEY, map(EMBEDDING_KEY, "vect", @Test public void queryVectorsWithSystemDbStorage() { String keyConfig = "chroma-config-foo"; - String baseUrl = "http://" + HOST; + String baseUrl = HOST; Map mapping = map(EMBEDDING_KEY, "vect", NODE_LABEL, "Test", ENTITY_KEY, "myId", diff --git a/extended-it/src/test/java/apoc/vectordb/QdrantTest.java b/extended-it/src/test/java/apoc/vectordb/QdrantTest.java index 1ff1bec6da..d11432c6f9 100644 --- a/extended-it/src/test/java/apoc/vectordb/QdrantTest.java +++ b/extended-it/src/test/java/apoc/vectordb/QdrantTest.java @@ -75,7 +75,7 @@ public static void setUp() throws Exception { QDRANT_CONTAINER.start(); - HOST = "localhost:" + QDRANT_CONTAINER.getMappedPort(6333); + HOST = QDRANT_CONTAINER.getHost() + ":" +QDRANT_CONTAINER.getMappedPort(6333); TestUtil.registerProcedure(db, Qdrant.class, VectorDb.class, Prompt.class); testCall(db, "CALL apoc.vectordb.qdrant.createCollection($host, 'test_collection', 'Cosine', 4, $conf)", diff --git a/extended/src/main/java/apoc/custom/CypherProceduresHandler.java b/extended/src/main/java/apoc/custom/CypherProceduresHandler.java index e13facd93d..d860cf1226 100644 --- a/extended/src/main/java/apoc/custom/CypherProceduresHandler.java +++ b/extended/src/main/java/apoc/custom/CypherProceduresHandler.java @@ -22,6 +22,7 @@ import org.neo4j.internal.kernel.api.procs.ProcedureSignature; import org.neo4j.internal.kernel.api.procs.QualifiedName; import org.neo4j.internal.kernel.api.procs.UserFunctionSignature; +import org.neo4j.kernel.api.CypherScope; import org.neo4j.kernel.api.ResourceMonitor; import org.neo4j.kernel.api.procedure.CallableProcedure; import org.neo4j.kernel.api.procedure.CallableUserFunction; @@ -31,7 +32,6 @@ import org.neo4j.kernel.internal.GraphDatabaseAPI; import org.neo4j.kernel.lifecycle.LifecycleAdapter; import org.neo4j.logging.Log; -import org.neo4j.procedure.Name; import org.neo4j.procedure.impl.ProcedureHolderUtils; import org.neo4j.scheduler.Group; import org.neo4j.scheduler.JobHandle; @@ -42,7 +42,6 @@ import org.neo4j.values.virtual.MapValueBuilder; import org.neo4j.values.virtual.VirtualValues; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -243,7 +242,7 @@ private long getLastUpdate() { public boolean registerProcedure(ProcedureSignature signature, String statement) { QualifiedName name = signature.name(); try { - boolean exists = globalProceduresRegistry.getCurrentView().getAllProcedures().stream() + boolean exists = globalProceduresRegistry.getCurrentView().getAllProcedures(CypherScope.CYPHER_5) .anyMatch(s -> s.name().equals(name)); if (exists) { // we deregister and remove possible homonyms signatures overridden/overloaded @@ -293,7 +292,7 @@ public boolean registerFunction(UserFunctionSignature signature) { public boolean registerFunction(UserFunctionSignature signature, String statement, boolean forceSingle, boolean mapResult) { try { QualifiedName name = signature.name(); - boolean exists = globalProceduresRegistry.getCurrentView().getAllNonAggregatingFunctions() + boolean exists = globalProceduresRegistry.getCurrentView().getAllNonAggregatingFunctions(CypherScope.CYPHER_5) .anyMatch(s -> s.name().equals(name)); if (exists) { // we deregister and remove possible homonyms signatures overridden/overloaded @@ -362,14 +361,6 @@ private boolean isWrapped(AnyType outType, boolean mapResult) { return !mapResult && outType.getClass().equals(Neo4jTypes.MapType.class); } - public static QualifiedName qualifiedName(@Name("name") String name) { - String[] names = name.split("\\."); - List namespace = new ArrayList<>(names.length); - namespace.add(PREFIX); - namespace.addAll(Arrays.asList(names)); - return new QualifiedName(namespace.subList(0, namespace.size() - 1), names[names.length - 1]); - } - private AnyValue[] toResult(Map row, String[] names, boolean defaultOutputs) { if (defaultOutputs) { return new AnyValue[]{convertToValueRecursive(row)}; diff --git a/extended/src/main/java/apoc/custom/CypherProceduresUtil.java b/extended/src/main/java/apoc/custom/CypherProceduresUtil.java index 27f9c98239..08efe95005 100644 --- a/extended/src/main/java/apoc/custom/CypherProceduresUtil.java +++ b/extended/src/main/java/apoc/custom/CypherProceduresUtil.java @@ -32,10 +32,12 @@ public class CypherProceduresUtil { public static QualifiedName qualifiedName(@Name("name") String name) { String[] names = name.split("\\."); - List namespace = new ArrayList<>(names.length); - namespace.add(PREFIX); - namespace.addAll(Arrays.asList(names)); - return new QualifiedName(namespace.subList(0, namespace.size() - 1), names[names.length - 1]); + List namespaceList = new ArrayList<>(names.length); + namespaceList.add(PREFIX); + namespaceList.addAll(Arrays.asList(names)); + String[] namespace = namespaceList.subList(0, namespaceList.size() - 1) + .toArray(String[]::new); + return new QualifiedName(namespace, names[names.length - 1]); } public static Mode mode(String s) { diff --git a/extended/src/main/java/apoc/custom/Signatures.java b/extended/src/main/java/apoc/custom/Signatures.java index 7fbcd82959..e036d6fbaf 100644 --- a/extended/src/main/java/apoc/custom/Signatures.java +++ b/extended/src/main/java/apoc/custom/Signatures.java @@ -94,15 +94,15 @@ public ProcedureSignature toProcedureSignature(SignatureParser.ProcedureContext return createProcedureSignature(name, inputSignatures, outputSignature, mode, admin, deprecated, description, warning, eager, caseInsensitive, false, false, false, false); } - public List namespace(SignatureParser.NamespaceContext namespaceContext) { + public String[] namespace(SignatureParser.NamespaceContext namespaceContext) { List parsed = namespaceContext == null ? Collections.emptyList() : namespaceContext.name().stream().map(this::name).collect(Collectors.toList()); if (prefix == null) { - return parsed; + return parsed.toArray(String[]::new); } else { ArrayList namespace = new ArrayList<>(); namespace.add(prefix); namespace.addAll(parsed); - return namespace; + return namespace.toArray(String[]::new); } } diff --git a/extended/src/main/java/apoc/cypher/CypherExtended.java b/extended/src/main/java/apoc/cypher/CypherExtended.java index c5c459ab08..6452d8153c 100644 --- a/extended/src/main/java/apoc/cypher/CypherExtended.java +++ b/extended/src/main/java/apoc/cypher/CypherExtended.java @@ -25,7 +25,6 @@ import org.neo4j.procedure.Procedure; import org.neo4j.procedure.TerminationGuard; -import java.io.IOException; import java.io.Reader; import java.util.ArrayList; import java.util.Collection; @@ -352,8 +351,8 @@ public RowResult(long row, Map result, String fileName) { private Reader readerForFile(@Name("file") String fileName) { try { return FileUtils.readerFor(fileName, CompressionAlgo.NONE.name(), urlAccessChecker); - } catch (IOException ioe) { - throw new RuntimeException("Error accessing file "+fileName,ioe); + } catch (Exception e) { + throw new RuntimeException("Error accessing file "+fileName,e); } } diff --git a/extended/src/main/java/apoc/export/arrow/ImportArrow.java b/extended/src/main/java/apoc/export/arrow/ImportArrow.java index ee3159f29e..7735087079 100644 --- a/extended/src/main/java/apoc/export/arrow/ImportArrow.java +++ b/extended/src/main/java/apoc/export/arrow/ImportArrow.java @@ -21,6 +21,7 @@ import org.neo4j.graphdb.Relationship; import org.neo4j.graphdb.RelationshipType; import org.neo4j.graphdb.security.URLAccessChecker; +import org.neo4j.graphdb.security.URLAccessValidationError; import org.neo4j.procedure.Context; import org.neo4j.procedure.Description; import org.neo4j.procedure.Mode; @@ -29,6 +30,7 @@ import java.io.ByteArrayInputStream; import java.io.IOException; +import java.net.URISyntaxException; import java.nio.channels.SeekableByteChannel; import java.util.Arrays; import java.util.Collection; @@ -152,7 +154,7 @@ public Stream importFile(@Name("input") Object input, @Name(value } - private ArrowReader getReader(Object input) throws IOException { + private ArrowReader getReader(Object input) throws IOException, URISyntaxException, URLAccessValidationError { RootAllocator allocator = new RootAllocator(); if (input instanceof String) { final SeekableByteChannel channel = FileUtils.inputStreamFor(input, null, null, null, urlAccessChecker) diff --git a/extended/src/main/java/apoc/export/parquet/ExportParquet.java b/extended/src/main/java/apoc/export/parquet/ExportParquet.java index f9d1319e8a..979508f55c 100644 --- a/extended/src/main/java/apoc/export/parquet/ExportParquet.java +++ b/extended/src/main/java/apoc/export/parquet/ExportParquet.java @@ -16,6 +16,8 @@ import org.neo4j.graphdb.Relationship; import org.neo4j.graphdb.Result; import org.neo4j.graphdb.Transaction; +import org.neo4j.graphdb.security.URLAccessChecker; +import org.neo4j.graphdb.security.URLAccessValidationError; import org.neo4j.logging.Log; import org.neo4j.procedure.Context; import org.neo4j.procedure.Name; @@ -57,6 +59,9 @@ public class ExportParquet { @Context public Pools pools; + @Context + public URLAccessChecker urlAccessChecker; + @Procedure("apoc.export.parquet.all.stream") @Description("Exports the full database as a Parquet byte array.") @@ -93,20 +98,20 @@ public Stream query(@Name("query") String query, @Name(value = @Procedure("apoc.export.parquet.all") @Description("Exports the full database as a Parquet file.") - public Stream all(@Name("file") String fileName, @Name(value = "config", defaultValue = "{}") Map config) throws IOException { + public Stream all(@Name("file") String fileName, @Name(value = "config", defaultValue = "{}") Map config) throws Exception { return exportParquet(fileName, new DatabaseSubGraph(tx), new ParquetConfig(config)); } @Procedure("apoc.export.parquet.data") @Description("Exports the given nodes and relationships as a Parquet file.") - public Stream data(@Name("nodes") List nodes, @Name("rels") List rels, @Name("file") String fileName, @Name(value = "config", defaultValue = "{}") Map config) throws IOException { + public Stream data(@Name("nodes") List nodes, @Name("rels") List rels, @Name("file") String fileName, @Name(value = "config", defaultValue = "{}") Map config) throws Exception { ParquetConfig conf = new ParquetConfig(config); return exportParquet(fileName, new NodesAndRelsSubGraph(tx, nodes, rels), conf); } @Procedure("apoc.export.parquet.graph") @Description("Exports the given graph as a Parquet file.") - public Stream graph(@Name("graph") Map graph, @Name("file") String fileName, @Name(value = "config", defaultValue = "{}") Map config) throws IOException { + public Stream graph(@Name("graph") Map graph, @Name("file") String fileName, @Name(value = "config", defaultValue = "{}") Map config) throws Exception { Collection nodes = (Collection) graph.get("nodes"); Collection rels = (Collection) graph.get("relationships"); ParquetConfig conf = new ParquetConfig(config); @@ -116,7 +121,7 @@ public Stream graph(@Name("graph") Map graph, @Name @Procedure("apoc.export.parquet.query") @Description("Exports the given Cypher query as a Parquet file.") - public Stream query(@Name("query") String query, @Name("file") String fileName, @Name(value = "config", defaultValue = "{}") Map config) throws IOException { + public Stream query(@Name("query") String query, @Name("file") String fileName, @Name(value = "config", defaultValue = "{}") Map config) throws Exception { ParquetConfig exportConfig = new ParquetConfig(config); Map params = config == null ? Collections.emptyMap() : (Map)config.getOrDefault("params", Collections.emptyMap()); Result result = tx.execute(query,params); @@ -124,12 +129,12 @@ public Stream query(@Name("query") String query, @Name("file") Str return exportParquet(fileName, result, exportConfig); } - public Stream exportParquet(String fileName, Object data, ParquetConfig config) throws IOException { + public Stream exportParquet(String fileName, Object data, ParquetConfig config) throws IOException, URLAccessValidationError { if (StringUtils.isBlank(fileName)) { throw new RuntimeException("The fileName must exists. Otherwise, use the `apoc.export.parquet.*.stream.` procedures to stream the export back to your client."); } // normalize file url - fileName = FileUtils.changeFileUrlIfImportDirectoryConstrained(fileName); + fileName = FileUtils.changeFileUrlIfImportDirectoryConstrained(fileName, urlAccessChecker); // we cannot use apocConfig().checkWriteAllowed(..) because the error is confusing // since it says "... use the `{stream:true}` config", but with arrow procedures the streaming mode is implemented via different procedures diff --git a/extended/src/main/java/apoc/export/parquet/ParquetReadUtil.java b/extended/src/main/java/apoc/export/parquet/ParquetReadUtil.java index 0a97c9135f..d3b4ce9c3f 100644 --- a/extended/src/main/java/apoc/export/parquet/ParquetReadUtil.java +++ b/extended/src/main/java/apoc/export/parquet/ParquetReadUtil.java @@ -8,9 +8,11 @@ import org.apache.parquet.io.SeekableInputStream; import org.apache.parquet.schema.LogicalTypeAnnotation; import org.neo4j.graphdb.security.URLAccessChecker; +import org.neo4j.graphdb.security.URLAccessValidationError; import java.io.ByteArrayInputStream; import java.io.IOException; +import java.net.URISyntaxException; import java.util.Map; import java.util.concurrent.TimeUnit; @@ -30,14 +32,14 @@ public static java.util.concurrent.TimeUnit toTimeUnitJava(LogicalTypeAnnotation }; } - public static InputFile getInputFile(Object source, URLAccessChecker urlAccessChecker) throws IOException { + public static InputFile getInputFile(Object source, URLAccessChecker urlAccessChecker) throws IOException, URISyntaxException, URLAccessValidationError { return new ParquetStream(FileUtils.inputStreamFor(source, null, null, CompressionAlgo.NONE.name(), urlAccessChecker).readAllBytes()); } public static ApocParquetReader getReader(Object source, ParquetConfig conf, URLAccessChecker urlAccessChecker) { try { return new ApocParquetReader(getInputFile(source, urlAccessChecker), conf); - } catch (IOException e) { + } catch (Exception e) { throw new RuntimeException(e); } } diff --git a/extended/src/main/java/apoc/load/LoadCsv.java b/extended/src/main/java/apoc/load/LoadCsv.java index f4c374a925..eaa74c60ea 100644 --- a/extended/src/main/java/apoc/load/LoadCsv.java +++ b/extended/src/main/java/apoc/load/LoadCsv.java @@ -57,7 +57,7 @@ public Stream csvParams(@Name("urlOrBinary") Object urlOrBinary, @Nam } reader = FileUtils.readerFor(urlOrBinary, httpHeaders, payload, config.getCompressionAlgo(), urlAccessChecker); return streamCsv(url, config, reader); - } catch (IOException e) { + } catch (Exception e) { closeReaderSafely(reader); if(!config.isFailOnError()) return Stream.of(new CSVResult(new String[0], new String[0], 0, true, Collections.emptyMap(), emptyList(), EnumSet.noneOf(Results.class))); diff --git a/extended/src/main/java/apoc/load/LoadDirectory.java b/extended/src/main/java/apoc/load/LoadDirectory.java index c73f81df7b..6b152bbdbb 100644 --- a/extended/src/main/java/apoc/load/LoadDirectory.java +++ b/extended/src/main/java/apoc/load/LoadDirectory.java @@ -12,6 +12,7 @@ import org.neo4j.graphdb.GraphDatabaseService; import org.neo4j.graphdb.Transaction; import org.neo4j.graphdb.security.URLAccessChecker; +import org.neo4j.graphdb.security.URLAccessValidationError; import org.neo4j.logging.Log; import org.neo4j.procedure.Context; import org.neo4j.procedure.Description; @@ -58,12 +59,12 @@ public Stream add(@Name("name") String na @Name("cypher") String cypher, @Name(value = "pattern", defaultValue = "*") String pattern, @Name(value = "urlDir", defaultValue = "") String urlDir, - @Name(value = "config", defaultValue = "{}") Map config) throws IOException { + @Name(value = "config", defaultValue = "{}") Map config) throws Exception { apocConfig().checkReadAllowed(urlDir, urlAccessChecker); Util.validateQuery(db, cypher, READ_WRITE, WRITE); LoadDirectoryItem.LoadDirectoryConfig conf = new LoadDirectoryItem.LoadDirectoryConfig(config); - LoadDirectoryItem loadDirectoryItem = new LoadDirectoryItem(name, pattern, cypher, checkIfUrlBlankAndGetFileUrl(urlDir), conf); + LoadDirectoryItem loadDirectoryItem = new LoadDirectoryItem(name, pattern, cypher, checkIfUrlBlankAndGetFileUrl(urlDir, urlAccessChecker), conf); loadDirectoryHandler.add(loadDirectoryItem); return loadDirectoryHandler.list(); @@ -91,13 +92,13 @@ public Stream list() { @Procedure @Description("apoc.load.directory('pattern', 'urlDir', {config}) YIELD value - Loads list of all files in the folder specified by the parameter urlDir satisfying the given pattern. If the parameter urlDir is not specified or empty, the files of the import folder are loaded instead.") - public Stream directory(@Name(value = "pattern", defaultValue = "*") String pattern, @Name(value = "urlDir", defaultValue = "") String urlDir, @Name(value = "config", defaultValue = "{}") Map config) throws IOException { + public Stream directory(@Name(value = "pattern", defaultValue = "*") String pattern, @Name(value = "urlDir", defaultValue = "") String urlDir, @Name(value = "config", defaultValue = "{}") Map config) throws Exception { if (urlDir == null) { throw new IllegalArgumentException("Invalid (null) urlDir"); } - urlDir = checkIfUrlBlankAndGetFileUrl(urlDir); + urlDir = checkIfUrlBlankAndGetFileUrl(urlDir, urlAccessChecker); boolean isRecursive = Util.toBoolean(config.getOrDefault("recursive", true)); @@ -114,13 +115,13 @@ public Stream directory(@Name(value = "pattern", defaultValue = "* } // visible for test purpose - public static String checkIfUrlBlankAndGetFileUrl(String urlDir) throws IOException { + public static String checkIfUrlBlankAndGetFileUrl(String urlDir, URLAccessChecker urlAccessChecker) throws IOException, URLAccessValidationError { if (StringUtils.isBlank(urlDir)) { final Path pathImport = Paths.get(ApocConfig.apocConfig().getImportDir()).toAbsolutePath(); // with replaceAll we remove final "/" from path return pathImport.toUri().toString().replaceAll(".$", ""); } - return FileUtils.changeFileUrlIfImportDirectoryConstrained(urlDir.replace("?", "%3F")); + return FileUtils.changeFileUrlIfImportDirectoryConstrained(urlDir.replace("?", "%3F"), urlAccessChecker); } } \ No newline at end of file diff --git a/extended/src/main/java/apoc/load/LoadHtml.java b/extended/src/main/java/apoc/load/LoadHtml.java index 63e5b8270e..fa119cb1d3 100644 --- a/extended/src/main/java/apoc/load/LoadHtml.java +++ b/extended/src/main/java/apoc/load/LoadHtml.java @@ -4,6 +4,8 @@ import apoc.result.MapResult; import apoc.util.MissingDependencyException; import apoc.util.FileUtils; + +import java.net.URISyntaxException; import java.nio.charset.UnsupportedCharsetException; import org.apache.commons.lang3.StringUtils; @@ -14,6 +16,7 @@ import org.jsoup.select.Elements; import org.neo4j.graphdb.GraphDatabaseService; import org.neo4j.graphdb.security.URLAccessChecker; +import org.neo4j.graphdb.security.URLAccessValidationError; import org.neo4j.logging.Log; import org.neo4j.procedure.Context; import org.neo4j.procedure.Description; @@ -97,7 +100,7 @@ private Stream readHtmlPage(String url, Map query, Ma } } - private InputStream getHtmlInputStream(String url, Map query, LoadHtmlConfig config) throws IOException { + private InputStream getHtmlInputStream(String url, Map query, LoadHtmlConfig config) throws IOException, URISyntaxException, URLAccessValidationError { final boolean isHeadless = config.isHeadless(); final boolean isAcceptInsecureCerts = config.isAcceptInsecureCerts(); diff --git a/extended/src/test/java/apoc/export/parquet/ParquetTest.java b/extended/src/test/java/apoc/export/parquet/ParquetTest.java index 170eb990f2..98275b333d 100644 --- a/extended/src/test/java/apoc/export/parquet/ParquetTest.java +++ b/extended/src/test/java/apoc/export/parquet/ParquetTest.java @@ -110,7 +110,7 @@ public void testStreamRoundtripParquetAllWithImportExportConfsDisabled() { apocConfig().setProperty(APOC_IMPORT_FILE_ENABLED, false); apocConfig().setProperty(APOC_EXPORT_FILE_ENABLED, false); - assertFails("CALL apoc.export.parquet.all('ignore.parquet')", EXPORT_TO_FILE_PARQUET_ERROR); + assertFails("CALL apoc.export.parquet.all('ignore.parquet')", LOAD_FROM_FILE_ERROR); } @Test diff --git a/extended/src/test/java/apoc/load/LoadCsvTest.java b/extended/src/test/java/apoc/load/LoadCsvTest.java index c4f672befd..51ed709762 100644 --- a/extended/src/test/java/apoc/load/LoadCsvTest.java +++ b/extended/src/test/java/apoc/load/LoadCsvTest.java @@ -331,7 +331,7 @@ public void testLoadCsvWithEscapedDelimiters() { } @Test public void testWithSpacesInFileName() throws Exception { - String url = "test pipe column with spaces in filename.csv"; + String url = "file:///test%20pipe%20column%20with%20spaces%20in%20filename.csv"; testResult(db, "CALL apoc.load.csv($url,{results:['map','list','stringMap','strings'],mapping:{name:{type:'string'},beverage:{array:true,arraySep:'|',type:'string'}}})", map("url",url), // 'file:test.csv' (r) -> { assertEquals(asList("Selma", asList("Soda")), r.next().get("list")); diff --git a/extended/src/test/java/apoc/systemdb/SystemDbTest.java b/extended/src/test/java/apoc/systemdb/SystemDbTest.java index 0199078439..0581d7869f 100644 --- a/extended/src/test/java/apoc/systemdb/SystemDbTest.java +++ b/extended/src/test/java/apoc/systemdb/SystemDbTest.java @@ -72,16 +72,18 @@ public void testGetGraph() throws Exception { Map map = Iterators.single(result); List nodes = (List) map.get("nodes"); List relationships = (List) map.get("relationships"); - assertEquals(6, nodes.size()); + assertEquals(7, nodes.size()); assertEquals(2, nodes.stream().filter( node -> "Database".equals(Iterables.single(node.getLabels()).name())).count()); assertEquals(2, nodes.stream().filter( node -> "DatabaseName".equals(Iterables.single(node.getLabels()).name())).count()); assertEquals(1, nodes.stream().filter( node -> "User".equals(Iterables.single(node.getLabels()).name())).count()); assertEquals(1, nodes.stream().filter( node -> "Version".equals(Iterables.single(node.getLabels()).name())).count()); + assertEquals(1, nodes.stream().filter( node -> "Auth".equals(Iterables.single(node.getLabels()).name())).count()); Set names = nodes.stream().map(node -> (String)node.getProperty("name")).filter(Objects::nonNull).collect(Collectors.toSet()); org.hamcrest.MatcherAssert.assertThat( names, Matchers.containsInAnyOrder("neo4j", "system")); - assertEquals( 2, relationships.size() ); + assertEquals( 3, relationships.size() ); assertEquals( 2, relationships.stream().filter( rel -> "TARGETS".equals( rel.getType().name() ) ).count() ); + assertEquals( 1, relationships.stream().filter( rel -> "HAS_AUTH".equals( rel.getType().name() ) ).count() ); }); } diff --git a/extended/src/test/java/apoc/util/ExtendedTestUtil.java b/extended/src/test/java/apoc/util/ExtendedTestUtil.java index e956f706e9..9abca82941 100644 --- a/extended/src/test/java/apoc/util/ExtendedTestUtil.java +++ b/extended/src/test/java/apoc/util/ExtendedTestUtil.java @@ -3,6 +3,7 @@ import org.neo4j.graphdb.GraphDatabaseService; import org.neo4j.graphdb.Result; import org.neo4j.graphdb.ResultTransformer; +import org.neo4j.graphdb.security.URLAccessChecker; import org.neo4j.test.assertion.Assert; import java.util.Collections; @@ -17,6 +18,13 @@ public class ExtendedTestUtil { + /** + * Mock URLAccessChecker instance with checkURL(URL url) {return url} + */ + public static class MockURLAccessChecker { + public static final URLAccessChecker INSTANCE = url -> url; + } + public static void assertMapEquals(Map expected, Map actual) { assertMapEquals(null, expected, actual); } diff --git a/extended/src/test/java/apoc/util/FileUtilsTest.java b/extended/src/test/java/apoc/util/FileUtilsTest.java index cfdb320564..f5ed03aaec 100644 --- a/extended/src/test/java/apoc/util/FileUtilsTest.java +++ b/extended/src/test/java/apoc/util/FileUtilsTest.java @@ -15,8 +15,10 @@ import java.net.URISyntaxException; import java.nio.file.Paths; +import static apoc.ApocConfig.APOC_IMPORT_FILE_ENABLED; import static apoc.ApocConfig.apocConfig; import static org.junit.Assert.assertEquals; +import static apoc.util.ExtendedTestUtil.MockURLAccessChecker; public class FileUtilsTest { @@ -54,6 +56,7 @@ public void setUp() throws Exception { if (testName.getMethodName().endsWith(TEST_WITH_DIRECTORY_IMPORT)) { apocConfig().setProperty("server.directories.import", importFolder); } + apocConfig().setProperty(APOC_IMPORT_FILE_ENABLED, true); TestUtil.registerProcedure(db, Config.class); } @@ -64,50 +67,49 @@ public void tearDown() { @Test public void notChangeFileUrlWithDirectoryImportConstrainedURI() throws Exception { - assertEquals(TEST_FILE_ABSOLUTE, FileUtils.changeFileUrlIfImportDirectoryConstrained(TEST_FILE)); + assertEquals(TEST_FILE_ABSOLUTE, FileUtils.changeFileUrlIfImportDirectoryConstrained(TEST_FILE, MockURLAccessChecker.INSTANCE)); } @Test public void notChangeFileUrlWithDirectoryAndProtocolImportConstrainedURI() throws Exception { - assertEquals(TEST_FILE_ABSOLUTE, FileUtils.changeFileUrlIfImportDirectoryConstrained("file://" + TEST_FILE)); + assertEquals(TEST_FILE_ABSOLUTE, FileUtils.changeFileUrlIfImportDirectoryConstrained("file://" + TEST_FILE, MockURLAccessChecker.INSTANCE)); } @Test public void changeNoSlashesUrlWithDirectoryImportConstrainedURIWithDirectoryImport() throws Exception { - assertEquals(TEST_FILE_RELATIVE, FileUtils.changeFileUrlIfImportDirectoryConstrained("test.csv")); + assertEquals(TEST_FILE_RELATIVE, FileUtils.changeFileUrlIfImportDirectoryConstrained("test.csv", MockURLAccessChecker.INSTANCE)); } @Test public void changeSlashUrlWithDirectoryImportConstrainedURIWithDirectoryImport() throws Exception { - assertEquals(TEST_FILE_RELATIVE, FileUtils.changeFileUrlIfImportDirectoryConstrained("/test.csv")); + assertEquals(TEST_FILE_RELATIVE, FileUtils.changeFileUrlIfImportDirectoryConstrained("/test.csv", MockURLAccessChecker.INSTANCE)); } @Test public void changeFileSlashUrlWithDirectoryImportConstrainedURIWithDirectoryImport() throws Exception { - assertEquals(TEST_FILE_RELATIVE, FileUtils.changeFileUrlIfImportDirectoryConstrained("file:/test.csv")); + assertEquals(TEST_FILE_RELATIVE, FileUtils.changeFileUrlIfImportDirectoryConstrained("file:/test.csv", MockURLAccessChecker.INSTANCE)); } @Test public void changeFileDoubleSlashesUrlWithDirectoryImportConstrainedURIWithDirectoryImport() throws Exception { - assertEquals(TEST_FILE_RELATIVE, FileUtils.changeFileUrlIfImportDirectoryConstrained("file://test.csv")); + assertEquals(TEST_FILE_RELATIVE, FileUtils.changeFileUrlIfImportDirectoryConstrained("file://test.csv", MockURLAccessChecker.INSTANCE)); } @Test public void changeFileTripleSlashesUrlWithDirectoryImportConstrainedURIWithDirectoryImport() throws Exception { - assertEquals(TEST_FILE_RELATIVE, FileUtils.changeFileUrlIfImportDirectoryConstrained("file:///test.csv")); + assertEquals(TEST_FILE_RELATIVE, FileUtils.changeFileUrlIfImportDirectoryConstrained("file:///test.csv", MockURLAccessChecker.INSTANCE)); } @Test public void importDirectoryWithRelativePathWithDirectoryImport() throws Exception { - assertEquals(TEST_FILE_RELATIVE, FileUtils.changeFileUrlIfImportDirectoryConstrained("test.csv")); + assertEquals(TEST_FILE_RELATIVE, FileUtils.changeFileUrlIfImportDirectoryConstrained("test.csv", MockURLAccessChecker.INSTANCE)); } - @Test public void importDirectoryWithRelativeArchivePathWithDirectoryImport() throws Exception { String localPath = "test.zip!sub/test.csv"; String expected = importFolder + "/" + localPath; - assertEquals(Paths.get(expected).toUri().toString(), FileUtils.changeFileUrlIfImportDirectoryConstrained(localPath)); + assertEquals(Paths.get(expected).toUri().toString(), FileUtils.changeFileUrlIfImportDirectoryConstrained(localPath, MockURLAccessChecker.INSTANCE)); } } diff --git a/gradle.properties b/gradle.properties new file mode 100644 index 0000000000..a58f2fe34b --- /dev/null +++ b/gradle.properties @@ -0,0 +1 @@ +org.gradle.jvmargs=-Xmx5G -XX:MaxMetaspaceSize=256m \ No newline at end of file