Solve dev 5.21 build errors (#4140)

* Try to solve 5.21 TC errors (#4137) * Try to solve 5.21 TC errors * de-ignored gradle.properties to be consistent with Core one * Solve 5.21 compile errors (#4126) * Solve 5.21 compile errors * Added CypherScope.CYPHER_5 * Fix compile 5.21 errors after restore submodule * solve error: cannot find symbol, due to org.neo4j.kernel.api.procedure.ProcedureView 5.21 changes * fix ParquetTest.testStreamRoundtripParquetAllWithImportExportConfsDisabled * fix LoadCsvTest.testWithSpacesInFileName * fix SystemDbTest.testGetGraph * Update .gitmodules to 5.21
neo4j-contrib · Jul 11, 2024 · c49fe8a · c49fe8a
1 parent 1ebbd99
commit c49fe8a
Show file tree

Hide file tree

Showing 20 changed files with 75 additions and 58 deletions.
diff --git a/.gitignore b/.gitignore
@@ -6,7 +6,6 @@ classes
 dependency-reduced-pom.xml
 derby*
 .gradle
-gradle.properties
 build/
 *~
 \#*

diff --git a/.gitmodules b/.gitmodules
@@ -1,4 +1,4 @@
 [submodule "apoc-core"]
 	path = apoc-core
 	url = https://github.com/neo4j/apoc
-	branch = 5.20
+	branch = 5.21
diff --git a/extended-it/src/test/java/apoc/vectordb/ChromaDbTest.java b/extended-it/src/test/java/apoc/vectordb/ChromaDbTest.java
@@ -69,7 +69,7 @@ public static void setUp() throws Exception {
 
         CHROMA_CONTAINER.start();
 
-        HOST = "localhost:" + CHROMA_CONTAINER.getMappedPort(8000);
+        HOST = CHROMA_CONTAINER.getEndpoint();
         TestUtil.registerProcedure(db, ChromaDb.class, VectorDb.class, Prompt.class);
 
         testCall(db, "CALL apoc.vectordb.chroma.createCollection($host, 'test_collection', 'cosine', 4)",
@@ -396,7 +396,7 @@ MAPPING_KEY, map(EMBEDDING_KEY, "vect",
     @Test
     public void queryVectorsWithSystemDbStorage() {
         String keyConfig = "chroma-config-foo";
-        String baseUrl = "http://" + HOST;
+        String baseUrl = HOST;
         Map<String, Object> mapping = map(EMBEDDING_KEY, "vect",
                 NODE_LABEL, "Test",
                 ENTITY_KEY, "myId",

diff --git a/extended-it/src/test/java/apoc/vectordb/QdrantTest.java b/extended-it/src/test/java/apoc/vectordb/QdrantTest.java
@@ -75,7 +75,7 @@ public static void setUp() throws Exception {
 
         QDRANT_CONTAINER.start();
 
-        HOST = "localhost:" + QDRANT_CONTAINER.getMappedPort(6333);
+        HOST = QDRANT_CONTAINER.getHost() + ":" +QDRANT_CONTAINER.getMappedPort(6333);
         TestUtil.registerProcedure(db, Qdrant.class, VectorDb.class, Prompt.class);
 
         testCall(db, "CALL apoc.vectordb.qdrant.createCollection($host, 'test_collection', 'Cosine', 4, $conf)",

diff --git a/extended/src/main/java/apoc/custom/CypherProceduresHandler.java b/extended/src/main/java/apoc/custom/CypherProceduresHandler.java
@@ -22,6 +22,7 @@
 import org.neo4j.internal.kernel.api.procs.ProcedureSignature;
 import org.neo4j.internal.kernel.api.procs.QualifiedName;
 import org.neo4j.internal.kernel.api.procs.UserFunctionSignature;
+import org.neo4j.kernel.api.CypherScope;
 import org.neo4j.kernel.api.ResourceMonitor;
 import org.neo4j.kernel.api.procedure.CallableProcedure;
 import org.neo4j.kernel.api.procedure.CallableUserFunction;
@@ -31,7 +32,6 @@
 import org.neo4j.kernel.internal.GraphDatabaseAPI;
 import org.neo4j.kernel.lifecycle.LifecycleAdapter;
 import org.neo4j.logging.Log;
-import org.neo4j.procedure.Name;
 import org.neo4j.procedure.impl.ProcedureHolderUtils;
 import org.neo4j.scheduler.Group;
 import org.neo4j.scheduler.JobHandle;
@@ -42,7 +42,6 @@
 import org.neo4j.values.virtual.MapValueBuilder;
 import org.neo4j.values.virtual.VirtualValues;
 
-import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
@@ -243,7 +242,7 @@ private long getLastUpdate() {
     public boolean registerProcedure(ProcedureSignature signature, String statement) {
         QualifiedName name = signature.name();
         try {
-            boolean exists = globalProceduresRegistry.getCurrentView().getAllProcedures().stream()
+            boolean exists = globalProceduresRegistry.getCurrentView().getAllProcedures(CypherScope.CYPHER_5)
                     .anyMatch(s -> s.name().equals(name));
             if (exists) {
                 // we deregister and remove possible homonyms signatures overridden/overloaded
@@ -293,7 +292,7 @@ public boolean registerFunction(UserFunctionSignature signature) {
     public boolean registerFunction(UserFunctionSignature signature, String statement, boolean forceSingle, boolean mapResult) {
         try {
             QualifiedName name = signature.name();
-            boolean exists = globalProceduresRegistry.getCurrentView().getAllNonAggregatingFunctions()
+            boolean exists = globalProceduresRegistry.getCurrentView().getAllNonAggregatingFunctions(CypherScope.CYPHER_5)
                     .anyMatch(s -> s.name().equals(name));
             if (exists) {
                 // we deregister and remove possible homonyms signatures overridden/overloaded
@@ -362,14 +361,6 @@ private boolean isWrapped(AnyType outType, boolean mapResult) {
         return !mapResult && outType.getClass().equals(Neo4jTypes.MapType.class);
     }
 
-    public static QualifiedName qualifiedName(@Name("name") String name) {
-        String[] names = name.split("\\.");
-        List<String> namespace = new ArrayList<>(names.length);
-        namespace.add(PREFIX);
-        namespace.addAll(Arrays.asList(names));
-        return new QualifiedName(namespace.subList(0, namespace.size() - 1), names[names.length - 1]);
-    }
-
     private AnyValue[] toResult(Map<String, Object> row, String[] names, boolean defaultOutputs) {
         if (defaultOutputs) {
             return new AnyValue[]{convertToValueRecursive(row)};

diff --git a/extended/src/main/java/apoc/custom/CypherProceduresUtil.java b/extended/src/main/java/apoc/custom/CypherProceduresUtil.java
@@ -32,10 +32,12 @@ public class CypherProceduresUtil {
 
     public static QualifiedName qualifiedName(@Name("name") String name) {
         String[] names = name.split("\\.");
-        List<String> namespace = new ArrayList<>(names.length);
-        namespace.add(PREFIX);
-        namespace.addAll(Arrays.asList(names));
-        return new QualifiedName(namespace.subList(0, namespace.size() - 1), names[names.length - 1]);
+        List<String> namespaceList = new ArrayList<>(names.length);
+        namespaceList.add(PREFIX);
+        namespaceList.addAll(Arrays.asList(names));
+        String[] namespace = namespaceList.subList(0, namespaceList.size() - 1)
+                .toArray(String[]::new);
+        return new QualifiedName(namespace, names[names.length - 1]);
     }
 
     public static Mode mode(String s) {

diff --git a/extended/src/main/java/apoc/custom/Signatures.java b/extended/src/main/java/apoc/custom/Signatures.java
@@ -94,15 +94,15 @@ public ProcedureSignature toProcedureSignature(SignatureParser.ProcedureContext
         return createProcedureSignature(name, inputSignatures, outputSignature, mode, admin, deprecated, description, warning, eager, caseInsensitive, false, false, false, false);
     }
 
-    public List<String> namespace(SignatureParser.NamespaceContext namespaceContext) {
+    public String[] namespace(SignatureParser.NamespaceContext namespaceContext) {
         List<String> parsed = namespaceContext == null ? Collections.emptyList() : namespaceContext.name().stream().map(this::name).collect(Collectors.toList());
         if (prefix == null) {
-            return parsed;
+            return parsed.toArray(String[]::new);
         } else {
             ArrayList<String> namespace = new ArrayList<>();
             namespace.add(prefix);
             namespace.addAll(parsed);
-            return namespace;
+            return namespace.toArray(String[]::new);
         }
     }
 

diff --git a/extended/src/main/java/apoc/cypher/CypherExtended.java b/extended/src/main/java/apoc/cypher/CypherExtended.java
@@ -25,7 +25,6 @@
 import org.neo4j.procedure.Procedure;
 import org.neo4j.procedure.TerminationGuard;
 
-import java.io.IOException;
 import java.io.Reader;
 import java.util.ArrayList;
 import java.util.Collection;
@@ -352,8 +351,8 @@ public RowResult(long row, Map<String, Object> result, String fileName) {
     private Reader readerForFile(@Name("file") String fileName) {
         try {
             return FileUtils.readerFor(fileName, CompressionAlgo.NONE.name(), urlAccessChecker);
-        } catch (IOException ioe) {
-            throw new RuntimeException("Error accessing file "+fileName,ioe);
+        } catch (Exception e) {
+            throw new RuntimeException("Error accessing file "+fileName,e);
         }
     }
 

diff --git a/extended/src/main/java/apoc/export/arrow/ImportArrow.java b/extended/src/main/java/apoc/export/arrow/ImportArrow.java
@@ -21,6 +21,7 @@
 import org.neo4j.graphdb.Relationship;
 import org.neo4j.graphdb.RelationshipType;
 import org.neo4j.graphdb.security.URLAccessChecker;
+import org.neo4j.graphdb.security.URLAccessValidationError;
 import org.neo4j.procedure.Context;
 import org.neo4j.procedure.Description;
 import org.neo4j.procedure.Mode;
@@ -29,6 +30,7 @@
 
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
+import java.net.URISyntaxException;
 import java.nio.channels.SeekableByteChannel;
 import java.util.Arrays;
 import java.util.Collection;
@@ -152,7 +154,7 @@ public Stream<ProgressInfo> importFile(@Name("input") Object input, @Name(value
     }
 
 
-    private ArrowReader getReader(Object input) throws IOException {
+    private ArrowReader getReader(Object input) throws IOException, URISyntaxException, URLAccessValidationError {
         RootAllocator allocator = new RootAllocator();
         if (input instanceof String) {
             final SeekableByteChannel channel = FileUtils.inputStreamFor(input, null, null, null, urlAccessChecker)

diff --git a/extended/src/main/java/apoc/export/parquet/ExportParquet.java b/extended/src/main/java/apoc/export/parquet/ExportParquet.java
@@ -16,6 +16,8 @@
 import org.neo4j.graphdb.Relationship;
 import org.neo4j.graphdb.Result;
 import org.neo4j.graphdb.Transaction;
+import org.neo4j.graphdb.security.URLAccessChecker;
+import org.neo4j.graphdb.security.URLAccessValidationError;
 import org.neo4j.logging.Log;
 import org.neo4j.procedure.Context;
 import org.neo4j.procedure.Name;
@@ -57,6 +59,9 @@ public class ExportParquet {
     @Context
     public Pools pools;
 
+    @Context
+    public URLAccessChecker urlAccessChecker;
+
 
     @Procedure("apoc.export.parquet.all.stream")
     @Description("Exports the full database as a Parquet byte array.")
@@ -93,20 +98,20 @@ public Stream<ByteArrayResult> query(@Name("query") String query, @Name(value =
 
     @Procedure("apoc.export.parquet.all")
     @Description("Exports the full database as a Parquet file.")
-    public Stream<ProgressInfo> all(@Name("file") String fileName, @Name(value = "config", defaultValue = "{}") Map<String, Object> config) throws IOException {
+    public Stream<ProgressInfo> all(@Name("file") String fileName, @Name(value = "config", defaultValue = "{}") Map<String, Object> config) throws Exception {
         return exportParquet(fileName, new DatabaseSubGraph(tx), new ParquetConfig(config));
     }
 
     @Procedure("apoc.export.parquet.data")
     @Description("Exports the given nodes and relationships as a Parquet file.")
-    public Stream<ProgressInfo> data(@Name("nodes") List<Node> nodes, @Name("rels") List<Relationship> rels, @Name("file") String fileName, @Name(value = "config", defaultValue = "{}") Map<String, Object> config) throws IOException {
+    public Stream<ProgressInfo> data(@Name("nodes") List<Node> nodes, @Name("rels") List<Relationship> rels, @Name("file") String fileName, @Name(value = "config", defaultValue = "{}") Map<String, Object> config) throws Exception {
         ParquetConfig conf = new ParquetConfig(config);
         return exportParquet(fileName, new NodesAndRelsSubGraph(tx, nodes, rels), conf);
     }
 
     @Procedure("apoc.export.parquet.graph")
     @Description("Exports the given graph as a Parquet file.")
-    public Stream<ProgressInfo> graph(@Name("graph") Map<String,Object> graph, @Name("file") String fileName, @Name(value = "config", defaultValue = "{}") Map<String, Object> config) throws IOException {
+    public Stream<ProgressInfo> graph(@Name("graph") Map<String,Object> graph, @Name("file") String fileName, @Name(value = "config", defaultValue = "{}") Map<String, Object> config) throws Exception {
         Collection<Node> nodes = (Collection<Node>) graph.get("nodes");
         Collection<Relationship> rels = (Collection<Relationship>) graph.get("relationships");
         ParquetConfig conf = new ParquetConfig(config);
@@ -116,20 +121,20 @@ public Stream<ProgressInfo> graph(@Name("graph") Map<String,Object> graph, @Name
 
     @Procedure("apoc.export.parquet.query")
     @Description("Exports the given Cypher query as a Parquet file.")
-    public Stream<ProgressInfo> query(@Name("query") String query, @Name("file") String fileName, @Name(value = "config", defaultValue = "{}") Map<String, Object> config) throws IOException {
+    public Stream<ProgressInfo> query(@Name("query") String query, @Name("file") String fileName, @Name(value = "config", defaultValue = "{}") Map<String, Object> config) throws Exception {
         ParquetConfig exportConfig = new ParquetConfig(config);
         Map<String,Object> params = config == null ? Collections.emptyMap() : (Map<String,Object>)config.getOrDefault("params", Collections.emptyMap());
         Result result = tx.execute(query,params);
 
         return exportParquet(fileName, result, exportConfig);
     }
 
-    public Stream<ProgressInfo> exportParquet(String fileName, Object data, ParquetConfig config) throws IOException {
+    public Stream<ProgressInfo> exportParquet(String fileName, Object data, ParquetConfig config) throws IOException, URLAccessValidationError {
         if (StringUtils.isBlank(fileName)) {
             throw new RuntimeException("The fileName must exists. Otherwise, use the `apoc.export.parquet.*.stream.` procedures to stream the export back to your client.");
         }
         // normalize file url
-        fileName = FileUtils.changeFileUrlIfImportDirectoryConstrained(fileName);
+        fileName = FileUtils.changeFileUrlIfImportDirectoryConstrained(fileName, urlAccessChecker);
 
         // we cannot use apocConfig().checkWriteAllowed(..) because the error is confusing
         //  since it says "... use the `{stream:true}` config", but with arrow procedures the streaming mode is implemented via different procedures

diff --git a/extended/src/main/java/apoc/export/parquet/ParquetReadUtil.java b/extended/src/main/java/apoc/export/parquet/ParquetReadUtil.java
@@ -8,9 +8,11 @@
 import org.apache.parquet.io.SeekableInputStream;
 import org.apache.parquet.schema.LogicalTypeAnnotation;
 import org.neo4j.graphdb.security.URLAccessChecker;
+import org.neo4j.graphdb.security.URLAccessValidationError;
 
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
+import java.net.URISyntaxException;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
 
@@ -30,14 +32,14 @@ public static java.util.concurrent.TimeUnit toTimeUnitJava(LogicalTypeAnnotation
         };
     }
 
-    public static InputFile getInputFile(Object source, URLAccessChecker urlAccessChecker) throws IOException {
+    public static InputFile getInputFile(Object source, URLAccessChecker urlAccessChecker) throws IOException, URISyntaxException, URLAccessValidationError {
         return new ParquetStream(FileUtils.inputStreamFor(source, null, null, CompressionAlgo.NONE.name(), urlAccessChecker).readAllBytes());
     }
 
     public static ApocParquetReader getReader(Object source, ParquetConfig conf, URLAccessChecker urlAccessChecker) {
         try {
             return new ApocParquetReader(getInputFile(source, urlAccessChecker), conf);
-        } catch (IOException e) {
+        } catch (Exception e) {
             throw new RuntimeException(e);
         }
     }

diff --git a/extended/src/main/java/apoc/load/LoadCsv.java b/extended/src/main/java/apoc/load/LoadCsv.java
@@ -57,7 +57,7 @@ public Stream<CSVResult> csvParams(@Name("urlOrBinary") Object urlOrBinary, @Nam
             }
             reader = FileUtils.readerFor(urlOrBinary, httpHeaders, payload, config.getCompressionAlgo(), urlAccessChecker);
             return streamCsv(url, config, reader);
-        } catch (IOException e) {
+        } catch (Exception e) {
             closeReaderSafely(reader);
             if(!config.isFailOnError())
                 return Stream.of(new CSVResult(new String[0], new String[0], 0, true, Collections.emptyMap(), emptyList(), EnumSet.noneOf(Results.class)));

diff --git a/extended/src/main/java/apoc/load/LoadDirectory.java b/extended/src/main/java/apoc/load/LoadDirectory.java
@@ -12,6 +12,7 @@
 import org.neo4j.graphdb.GraphDatabaseService;
 import org.neo4j.graphdb.Transaction;
 import org.neo4j.graphdb.security.URLAccessChecker;
+import org.neo4j.graphdb.security.URLAccessValidationError;
 import org.neo4j.logging.Log;
 import org.neo4j.procedure.Context;
 import org.neo4j.procedure.Description;
@@ -58,12 +59,12 @@ public Stream<LoadDirectoryItem.LoadDirectoryResult> add(@Name("name") String na
                                                              @Name("cypher") String cypher,
                                                              @Name(value = "pattern", defaultValue = "*") String pattern,
                                                              @Name(value = "urlDir", defaultValue = "") String urlDir,
-                                                             @Name(value = "config", defaultValue = "{}") Map<String, Object> config) throws IOException {
+                                                             @Name(value = "config", defaultValue = "{}") Map<String, Object> config) throws Exception {
         apocConfig().checkReadAllowed(urlDir, urlAccessChecker);
         Util.validateQuery(db, cypher, READ_WRITE, WRITE);
 
         LoadDirectoryItem.LoadDirectoryConfig conf = new LoadDirectoryItem.LoadDirectoryConfig(config);
-        LoadDirectoryItem loadDirectoryItem = new LoadDirectoryItem(name, pattern, cypher, checkIfUrlBlankAndGetFileUrl(urlDir), conf);
+        LoadDirectoryItem loadDirectoryItem = new LoadDirectoryItem(name, pattern, cypher, checkIfUrlBlankAndGetFileUrl(urlDir, urlAccessChecker), conf);
 
         loadDirectoryHandler.add(loadDirectoryItem);
         return loadDirectoryHandler.list();
@@ -91,13 +92,13 @@ public Stream<LoadDirectoryItem.LoadDirectoryResult> list() {
 
     @Procedure
     @Description("apoc.load.directory('pattern', 'urlDir', {config}) YIELD value - Loads list of all files in the folder specified by the parameter urlDir satisfying the given pattern. If the parameter urlDir is not specified or empty, the files of the import folder are loaded instead.")
-    public Stream<StringResult> directory(@Name(value = "pattern", defaultValue = "*") String pattern, @Name(value = "urlDir", defaultValue = "") String urlDir, @Name(value = "config", defaultValue = "{}") Map<String, Object> config) throws IOException {
+    public Stream<StringResult> directory(@Name(value = "pattern", defaultValue = "*") String pattern, @Name(value = "urlDir", defaultValue = "") String urlDir, @Name(value = "config", defaultValue = "{}") Map<String, Object> config) throws Exception {
 
         if (urlDir == null) {
             throw new IllegalArgumentException("Invalid (null) urlDir");
         }
 
-        urlDir = checkIfUrlBlankAndGetFileUrl(urlDir);
+        urlDir = checkIfUrlBlankAndGetFileUrl(urlDir, urlAccessChecker);
 
         boolean isRecursive = Util.toBoolean(config.getOrDefault("recursive", true));
 
@@ -114,13 +115,13 @@ public Stream<StringResult> directory(@Name(value = "pattern", defaultValue = "*
     }
 
     // visible for test purpose
-    public static String checkIfUrlBlankAndGetFileUrl(String urlDir) throws IOException {
+    public static String checkIfUrlBlankAndGetFileUrl(String urlDir, URLAccessChecker urlAccessChecker) throws IOException, URLAccessValidationError {
         if (StringUtils.isBlank(urlDir)) {
             final Path pathImport = Paths.get(ApocConfig.apocConfig().getImportDir()).toAbsolutePath();
             // with replaceAll we remove final "/" from path
             return pathImport.toUri().toString().replaceAll(".$", "");
         }
-        return FileUtils.changeFileUrlIfImportDirectoryConstrained(urlDir.replace("?", "%3F"));
+        return FileUtils.changeFileUrlIfImportDirectoryConstrained(urlDir.replace("?", "%3F"), urlAccessChecker);
     }
 
 }
diff --git a/extended/src/main/java/apoc/load/LoadHtml.java b/extended/src/main/java/apoc/load/LoadHtml.java
@@ -4,6 +4,8 @@
 import apoc.result.MapResult;
 import apoc.util.MissingDependencyException;
 import apoc.util.FileUtils;
+
+import java.net.URISyntaxException;
 import java.nio.charset.UnsupportedCharsetException;
 
 import org.apache.commons.lang3.StringUtils;
@@ -14,6 +16,7 @@
 import org.jsoup.select.Elements;
 import org.neo4j.graphdb.GraphDatabaseService;
 import org.neo4j.graphdb.security.URLAccessChecker;
+import org.neo4j.graphdb.security.URLAccessValidationError;
 import org.neo4j.logging.Log;
 import org.neo4j.procedure.Context;
 import org.neo4j.procedure.Description;
@@ -97,7 +100,7 @@ private Stream<MapResult> readHtmlPage(String url, Map<String, String> query, Ma
         }
     }
 
-    private InputStream getHtmlInputStream(String url, Map<String, String> query, LoadHtmlConfig config) throws IOException {
+    private InputStream getHtmlInputStream(String url, Map<String, String> query, LoadHtmlConfig config) throws IOException, URISyntaxException, URLAccessValidationError {
 
         final boolean isHeadless = config.isHeadless();
         final boolean isAcceptInsecureCerts = config.isAcceptInsecureCerts();

diff --git a/extended/src/test/java/apoc/export/parquet/ParquetTest.java b/extended/src/test/java/apoc/export/parquet/ParquetTest.java
@@ -110,7 +110,7 @@ public void testStreamRoundtripParquetAllWithImportExportConfsDisabled() {
         apocConfig().setProperty(APOC_IMPORT_FILE_ENABLED, false);
         apocConfig().setProperty(APOC_EXPORT_FILE_ENABLED, false);
 
-        assertFails("CALL apoc.export.parquet.all('ignore.parquet')", EXPORT_TO_FILE_PARQUET_ERROR);
+        assertFails("CALL apoc.export.parquet.all('ignore.parquet')", LOAD_FROM_FILE_ERROR);
     }
 
     @Test