Cypher25 APOC proc func migration (#4208)

* [lYfZxdRz] Migrate procedures and functions from Core to Cypher 25 --------- Co-authored-by: Omar Ahmad <[email protected]>
neo4j-contrib · Oct 28, 2024 · 9121a81 · 9121a81
1 parent 4f0586f
commit 9121a81
Show file tree

Hide file tree

Showing 47 changed files with 2,969 additions and 130 deletions.
diff --git a/.github/actions/test-gradle-project/action.yml b/.github/actions/test-gradle-project/action.yml
@@ -10,16 +10,16 @@ runs:
   using: "composite"
   steps:
     - uses: ./.github/actions/setup-gradle-cache
-    - name: Run compile tests 
+    - name: Run compile tests
       shell: bash
-      run: ./gradlew :${{inputs.project-name}}:compileJava :${{inputs.project-name}}:compileTestJava
+      run: ./gradlew --info -Pneo4jVersionOverride=$NEO4JVERSION -Pneo4jDockerEeOverride=$NEO4J_DOCKER_EE_OVERRIDE -Pneo4jDockerCeOverride=$NEO4J_DOCKER_CE_OVERRIDE :${{inputs.project-name}}:compileJava :${{inputs.project-name}}:compileTestJava
     - name: Run tests
       shell: bash
-      run: ./gradlew :${{inputs.project-name}}:check --parallel
+      run: ./gradlew --info -Pneo4jVersionOverride=$NEO4JVERSION -Pneo4jDockerEeOverride=$NEO4J_DOCKER_EE_OVERRIDE -Pneo4jDockerCeOverride=$NEO4J_DOCKER_CE_OVERRIDE :${{inputs.project-name}}:check --parallel
     - name: Archive test results
       uses: actions/upload-artifact@v3
       if: always()
       with:
         name: ${{inputs.project-name}}-test-results
         path: |
-          ${{inputs.project-name}}/build/reports/tests/test/
+          ${{inputs.project-name}}/build/reports/tests/test/
diff --git a/.github/workflows/CI.yaml b/.github/workflows/CI.yaml
@@ -9,7 +9,8 @@ on:
 env:
   CODEARTIFACT_DOWNLOAD_URL: ${{ secrets.CODEARTIFACT_DOWNLOAD_URL }}
   CODEARTIFACT_USERNAME: ${{ secrets.CODEARTIFACT_USERNAME }}
-
+  ECR_NEO4J_DOCKER_URL: ${{ secrets.ECR_NEO4J_DOCKER_URL }}
+
 
 jobs:
   code-ql:
@@ -23,23 +24,53 @@ jobs:
       matrix:
         language: [ 'java', 'javascript' ]
     steps:
+      - name: Configure AWS CLI
+        uses: aws-actions/configure-aws-credentials@v2
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: eu-west-1
+
+      - name: Configure CodeArtifact Authentication Token
+        run: |
+          CODEARTIFACT_TOKEN=`aws codeartifact get-authorization-token --domain build-service-live --domain-owner ${{ secrets.AWS_ACCOUNT_ID }} --query authorizationToken --output text`
+          echo "::add-mask::$CODEARTIFACT_TOKEN"
+          echo "CODEARTIFACT_TOKEN=$CODEARTIFACT_TOKEN" >> "$GITHUB_ENV"
+      
+      - name: Login in to AWS ECR
+        run: |
+          aws ecr get-login-password --region eu-west-1 | docker login --username AWS --password-stdin $ECR_NEO4J_DOCKER_URL
+
       - uses: actions/checkout@v3
       - uses: ./.github/actions/setup-jdk
       - uses: ./.github/actions/setup-gradle-cache
-
+
+      - name: Determine latest neo4j CI version and docker nightly images
+        run: |
+          neo4j_version_base=$(grep -e "neo4jVersion = .*" build.gradle | cut -d '=' -f 2 | tr -d \'\" | tr -d ' ')
+          echo "neo4j_version_base=$neo4j_version_base"
+          NEO4JVERSION=`aws codeartifact list-package-versions --domain build-service-live --domain-owner ${{ secrets.AWS_ACCOUNT_ID }} --repository ci-live --format maven --namespace org.neo4j --package neo4j --sort-by PUBLISHED_TIME --query "versions[?starts_with(version,'$neo4j_version_base')] | [0].version" | tr -d '" '`
+          echo "NEO4JVERSION=$NEO4JVERSION" >> "$GITHUB_ENV"
+          echo "Found NEO4JVERSION=$NEO4JVERSION"
+          NEO4J_DOCKER_EE_OVERRIDE="$ECR_NEO4J_DOCKER_URL:$neo4j_version_base-enterprise-debian-nightly"
+          echo "NEO4J_DOCKER_EE_OVERRIDE=$NEO4J_DOCKER_EE_OVERRIDE" >> "$GITHUB_ENV"
+          echo "Found NEO4J_DOCKER_EE_OVERRIDE=$NEO4J_DOCKER_EE_OVERRIDE"
+          NEO4J_DOCKER_CE_OVERRIDE="$ECR_NEO4J_DOCKER_URL:$neo4j_version_base-community-debian-nightly"
+          echo "NEO4J_DOCKER_CE_OVERRIDE=$NEO4J_DOCKER_CE_OVERRIDE" >> "$GITHUB_ENV"
+          echo "Found NEO4J_DOCKER_CE_OVERRIDE=$NEO4J_DOCKER_CE_OVERRIDE"
+
       - name: Compile Java
         run: |
           chmod +x gradlew
-          ./gradlew --no-daemon --init-script init.gradle clean
+          ./gradlew --no-daemon --info -Pneo4jVersionOverride=$NEO4JVERSION --init-script init.gradle clean
       # Initializes the CodeQL tools for scanning.
       - name: Initialize CodeQL
         uses: github/codeql-action/init@v2
         with:
           languages: ${{ matrix.language }}
-
-      # Autobuild attempts to build any compiled languages 
-      - name: Autobuild
-        uses: github/codeql-action/autobuild@v2
+
+      - name: Compile
+        run: ./gradlew --info -Pneo4jVersionOverride=$NEO4JVERSION compileJava compileTestJava
 
       - name: Perform CodeQL Analysis
         uses: github/codeql-action/analyze@v2
@@ -53,6 +84,23 @@ jobs:
         project: [ 'extended', 'extended-it' ]
     runs-on: ubuntu-latest
     steps:
+      - name: Configure AWS CLI
+        uses: aws-actions/configure-aws-credentials@v2
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: eu-west-1
+
+      - name: Configure CodeArtifact Authentication Token
+        run: |
+          CODEARTIFACT_TOKEN=`aws codeartifact get-authorization-token --domain build-service-live --domain-owner ${{ secrets.AWS_ACCOUNT_ID }} --query authorizationToken --output text`
+          echo "::add-mask::$CODEARTIFACT_TOKEN"
+          echo "CODEARTIFACT_TOKEN=$CODEARTIFACT_TOKEN" >> "$GITHUB_ENV"
+      
+      - name: Login in to AWS ECR
+        run: |
+          aws ecr get-login-password --region eu-west-1 | docker login --username AWS --password-stdin $ECR_NEO4J_DOCKER_URL
+
       - uses: actions/checkout@v2
       - name: Set up JDK 17
         uses: actions/setup-java@v2
@@ -66,10 +114,24 @@ jobs:
             ~/.gradle/wrapper
           key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}
 
+      - name: Determine latest neo4j CI version and docker nightly images
+        run: |
+          neo4j_version_base=$(grep -e "neo4jVersion = .*" build.gradle | cut -d '=' -f 2 | tr -d \'\" | tr -d ' ')
+          echo "neo4j_version_base=$neo4j_version_base"
+          NEO4JVERSION=`aws codeartifact list-package-versions --domain build-service-live --domain-owner ${{ secrets.AWS_ACCOUNT_ID }} --repository ci-live --format maven --namespace org.neo4j --package neo4j --sort-by PUBLISHED_TIME --query "versions[?starts_with(version,'$neo4j_version_base')] | [0].version" | tr -d '" '`
+          echo "NEO4JVERSION=$NEO4JVERSION" >> "$GITHUB_ENV"
+          echo "Found NEO4JVERSION=$NEO4JVERSION"
+          NEO4J_DOCKER_EE_OVERRIDE="$ECR_NEO4J_DOCKER_URL/build-service/neo4j:$neo4j_version_base-enterprise-debian-nightly"
+          echo "NEO4J_DOCKER_EE_OVERRIDE=$NEO4J_DOCKER_EE_OVERRIDE" >> "$GITHUB_ENV"
+          echo "Found NEO4J_DOCKER_EE_OVERRIDE=$NEO4J_DOCKER_EE_OVERRIDE"
+          NEO4J_DOCKER_CE_OVERRIDE="$ECR_NEO4J_DOCKER_URL/build-service/neo4j:$neo4j_version_base-community-debian-nightly"
+          echo "NEO4J_DOCKER_CE_OVERRIDE=$NEO4J_DOCKER_CE_OVERRIDE" >> "$GITHUB_ENV"
+          echo "Found NEO4J_DOCKER_CE_OVERRIDE=$NEO4J_DOCKER_CE_OVERRIDE"
+
       - name: Init gradle
         run: |
           chmod +x gradlew
-          ./gradlew --init-script init.gradle
+          ./gradlew --info -Pneo4jVersionOverride=$NEO4JVERSION -Pneo4jDockerEeOverride=$NEO4J_DOCKER_EE_OVERRIDE -Pneo4jDockerCeOverride=$NEO4J_DOCKER_CE_OVERRIDE --init-script init.gradle
 
       - name: Run ${{ matrix.project }} tests
         uses: ./.github/actions/test-gradle-project

diff --git a/.gitmodules b/.gitmodules
@@ -1,4 +1,4 @@
 [submodule "apoc-core"]
 	path = apoc-core
 	url = https://github.com/neo4j/apoc
-	branch = 5.24
+	branch = dev
diff --git a/build.gradle b/build.gradle
@@ -4,6 +4,7 @@ plugins {
     id 'maven-publish'
     id 'antlr'
     id "com.github.hierynomus.license-report" version"0.16.1"
+    id "org.jetbrains.kotlin.jvm" version "1.8.0" apply false
 }
 
 downloadLicenses {
@@ -14,7 +15,7 @@ downloadLicenses {
 
 allprojects {
     group = 'org.neo4j.procedure'
-    version = '5.24.0'
+    version = '5.26.0'
     archivesBaseName = 'apoc'
     description = """neo4j-apoc-procedures"""
 }
@@ -28,7 +29,17 @@ repositories {
     /*maven {  // this contains the neo4j 4.0.0-beta jars
     url "https://neo4j.bintray.com/community/"
 }*/
-    mavenCentral()
+    if (System.getenv("CODEARTIFACT_DOWNLOAD_URL") ?: "" != "") {
+        maven {
+            url System.getenv('CODEARTIFACT_DOWNLOAD_URL')
+            credentials {
+                username System.getenv('CODEARTIFACT_USERNAME')
+                password System.getenv('CODEARTIFACT_TOKEN')
+            }
+        }
+    } else {
+        mavenCentral()
+    }
     maven {
         url "https://repo.gradle.org/gradle/libs-releases"
     }
@@ -43,7 +54,17 @@ subprojects {
         /*maven {  // this contains the neo4j 4.0.0-beta jars
         url "https://neo4j.bintray.com/community/"
     }*/
-        mavenCentral()
+        if (System.getenv("CODEARTIFACT_DOWNLOAD_URL") ?: "" != "") {
+            maven {
+                url System.getenv('CODEARTIFACT_DOWNLOAD_URL')
+                credentials {
+                    username System.getenv('CODEARTIFACT_USERNAME')
+                    password System.getenv('CODEARTIFACT_TOKEN')
+                }
+            }
+        } else {
+            mavenCentral()
+        }
         maven {
             url "https://repo.gradle.org/gradle/libs-releases"
         }
@@ -58,7 +79,7 @@ subprojects {
 
     task mySourcesJar(type: Jar) {
         from sourceSets.main.allJava
-        classifier = 'sources'
+        archiveClassifier = 'sources'
     }
 
     task myJavadocJar(type: Jar) {
@@ -71,8 +92,8 @@ subprojects {
         // neo4jDockerImage system property is used in TestContainerUtil
         systemProperties 'user.language' : 'en' ,
                 'user.country' : 'US',
-                'neo4jDockerImage' : System.getProperty("NEO4JVERSION") ? 'neo4j:' + System.getProperty("NEO4JVERSION") + '-enterprise-debian' : 'neo4j:5.24.1-enterprise',
-                'neo4jCommunityDockerImage': System.getProperty("NEO4JVERSION") ? 'neo4j:' + System.getProperty("NEO4JVERSION") + '-debian' : 'neo4j:5.24.1',
+                'neo4jDockerImage': project.hasProperty("neo4jDockerEeOverride") ? project.getProperty("neo4jDockerEeOverride") : 'neo4j:5.26.0-enterprise',
+                'neo4jCommunityDockerImage': project.hasProperty("neo4jDockerCeOverride") ? project.getProperty("neo4jDockerCeOverride") : 'neo4j:5.26.0',
                 'coreDir': 'apoc-core/core',
                 'testDockerBundle': false
 
@@ -131,9 +152,9 @@ subprojects {
 
 ext {
     // NB: due to version.json generation by parsing this file, the next line must not have any if/then/else logic
-    neo4jVersion = "5.24.1"
+    neo4jVersion = "5.26.0"
     // instead we apply the override logic here
-    neo4jVersionEffective = project.hasProperty("neo4jVersionOverride") ? project.getProperty("neo4jVersionOverride") : neo4jVersion
-    testContainersVersion = '1.18.3'
+    neo4jVersionEffective = project.hasProperty("neo4jVersionOverride") ? project.getProperty("neo4jVersionOverride") : neo4jVersion + "-SNAPSHOT"
+    testContainersVersion = '1.20.2'
     apacheArrowVersion = '15.0.0'
-}
+}
diff --git a/docs/asciidoc/modules/ROOT/pages/ml/openai.adoc b/docs/asciidoc/modules/ROOT/pages/ml/openai.adoc
@@ -18,7 +18,7 @@ All the following procedures can have the following APOC config, i.e. in `apoc.c
 | apoc.ml.openai.url | the OpenAI endpoint base url | `https://api.openai.com/v1` by default, 
     `https://api.anthropic.com/v1` if `apoc.ml.openai.type=<ANTHROPIC>`,
     or empty string if `apoc.ml.openai.type=<AZURE OR HUGGINGFACE>`
-| apoc.ml.azure.api.version | in case of `apoc.ml.openai.type=AZURE`, indicates the `api-version` to be passed after the `?api-version=` url
+| apoc.ml.azure.api.version | in case of `apoc.ml.openai.type=AZURE`, indicates the `api-version` to be passed after the `?api-version=` url | ""
 |===
 
 

diff --git a/extended-it/build.gradle b/extended-it/build.gradle
@@ -3,7 +3,7 @@ plugins {
     id 'com.github.johnrengelman.shadow'
     id 'maven-publish'
     id 'antlr'
-    id "org.jetbrains.kotlin.jvm" version "1.6.0"
+    id "org.jetbrains.kotlin.jvm" version "1.8.0"
     id "com.diffplug.spotless" version "6.7.2"
 }
 
@@ -51,10 +51,10 @@ dependencies {
     }
     testImplementation group: 'org.apache.parquet', name: 'parquet-hadoop', version: '1.13.1', withoutServers
 
-    testImplementation group: 'org.testcontainers', name: 'qdrant', version: '1.19.7'
-    testImplementation group: 'org.testcontainers', name: 'chromadb', version: '1.19.7'
-    testImplementation group: 'org.testcontainers', name: 'weaviate', version: '1.19.7'
-    testImplementation group: 'org.testcontainers', name: 'milvus', version: '1.19.7'
+    testImplementation group: 'org.testcontainers', name: 'qdrant', version: '1.20.2'
+    testImplementation group: 'org.testcontainers', name: 'chromadb', version: '1.20.2'
+    testImplementation group: 'org.testcontainers', name: 'weaviate', version: '1.20.2'
+    testImplementation group: 'org.testcontainers', name: 'milvus', version: '1.20.2'
 
     configurations.all {
         exclude group: 'org.slf4j', module: 'slf4j-nop'

diff --git a/extended-it/src/test/java/apoc/MissingExtraDependenciesTest.java b/extended-it/src/test/java/apoc/MissingExtraDependenciesTest.java
@@ -47,29 +47,6 @@ public static void tearDown() {
         neo4jContainer.close();
     }
 
-    @Test
-    public void testParquet() {
-        // export file
-        assertParquetFails("CALL apoc.export.parquet.all('test.parquet')");
-        assertParquetFails("CALL apoc.export.parquet.data([], [], 'test.parquet')");
-        assertParquetFails("CALL apoc.export.parquet.graph({nodes: [], relationships: []}, 'test.parquet')");
-        assertParquetFails("CALL apoc.export.parquet.query('MATCH (n:ParquetNode) RETURN n', 'test.parquet')");
-
-        // export stream
-        assertParquetFails("CALL apoc.export.parquet.all.stream()");
-        assertParquetFails("CALL apoc.export.parquet.data.stream([], [])");
-        assertParquetFails("CALL apoc.export.parquet.graph.stream({nodes: [], relationships: []})");
-        assertParquetFails("CALL apoc.export.parquet.query.stream('MATCH (n:ParquetNode) RETURN n')");
-
-        // import and load
-        assertParquetFails("CALL apoc.import.parquet('test.parquet')");
-        assertParquetFails("CALL apoc.load.parquet('test.parquet')");
-    }
-
-    private static void assertParquetFails(String query) {
-        assertFails(query, PARQUET_MISSING_DEPS_ERROR);
-    }
-
     @Test
     public void testCouchbase() {
         assertCouchbaseFails("CALL apoc.couchbase.get('host', 'bucket', 'documentId')");

diff --git a/extended/build.gradle b/extended/build.gradle
@@ -2,11 +2,11 @@ import org.gradle.api.internal.artifacts.DefaultExcludeRule
 
 plugins {
     id 'java'
-    id 'com.github.johnrengelman.shadow'
+    id 'com.github.johnrengelman.shadow' version '7.1.0'
     id 'maven-publish'
     id 'antlr'
-    id "org.jetbrains.kotlin.jvm" version "1.6.0"
-    id "com.diffplug.spotless" version "6.7.2"
+    id "org.jetbrains.kotlin.jvm" version "1.8.0"
+    id "com.diffplug.spotless" version "6.22.0"
 }
 
 spotless {
@@ -26,13 +26,19 @@ jar {
 }
 
 compileKotlin {
+    dependsOn(generateGrammarSource)
     kotlinOptions.jvmTarget = "17"
 }
 
 generateGrammarSource {
     arguments += ["-package", "apoc.custom"]
 }
 
+sourceSets.configureEach {
+    var generateGrammarSource = tasks.named(getTaskName("generate", "GrammarSource"))
+    java.srcDir(generateGrammarSource.map { files() })
+}
+
 javadoc {
     failOnError = false
     options.addStringOption('Xdoclint:none', '-quiet')
@@ -54,11 +60,13 @@ task gitSubmoduleLoad {
 dependencies {
     // to fix unresolved imports with IntelliJ from version 2024.1.x
     compileOnly sourceSets.main.java
+
+
 
     apt project(':processor')
     apt group: 'org.neo4j', name: 'neo4j', version: neo4jVersionEffective   // mandatory to run @ServiceProvider based META-INF code generation
 
-    antlr "org.antlr:antlr4:4.13.1", {
+    antlr "org.antlr:antlr4:4.13.2", {
         exclude group: 'org.glassfish'
         exclude group: 'com.ibm.icu'
         exclude group: 'org.abego.treelayout'
@@ -111,7 +119,7 @@ dependencies {
     compileOnly group: 'com.amazonaws', name: 'aws-java-sdk-s3', version: '1.11.270'
     compileOnly group: 'com.amazonaws', name: 'aws-java-sdk-comprehend', version: '1.12.353' , withoutJacksons
     compileOnly group: 'com.sun.mail', name: 'javax.mail', version: '1.6.0'
-    compileOnly group: 'org.jetbrains.kotlin', name: 'kotlin-stdlib-jdk8', version: '1.6.0'
+    compileOnly group: 'org.jetbrains.kotlin', name: 'kotlin-stdlib-jdk8', version: '1.8.0'
 
     compileOnly group: 'org.apache.parquet', name: 'parquet-hadoop', version: '1.13.1', withoutServers
     // testImplementation analogous is not needed since is bundled via `test-utils` submodule
@@ -133,7 +141,7 @@ dependencies {
     testImplementation group: 'com.github.stefanbirkner', name: 'system-rules', version: '1.19.0'
     testImplementation group: 'com.amazonaws', name: 'aws-java-sdk-s3', version: '1.11.270'
     testImplementation group: 'org.reflections', name: 'reflections', version: '0.9.12'
-    testImplementation group: 'org.jetbrains.kotlin', name: 'kotlin-stdlib-jdk8', version: '1.6.0'
+    testImplementation group: 'org.jetbrains.kotlin', name: 'kotlin-stdlib-jdk8', version: '1.8.0'
     testImplementation group: 'org.hamcrest', name: 'hamcrest-library', version: '1.3'
     testImplementation group: 'org.apache.derby', name: 'derby', version: '10.12.1.1'
     testImplementation group: 'org.mock-server', name: 'mockserver-netty', version: '5.6.0'

diff --git a/extended/src/main/java/apoc/custom/CypherProceduresHandler.java b/extended/src/main/java/apoc/custom/CypherProceduresHandler.java
@@ -22,7 +22,7 @@
 import org.neo4j.internal.kernel.api.procs.ProcedureSignature;
 import org.neo4j.internal.kernel.api.procs.QualifiedName;
 import org.neo4j.internal.kernel.api.procs.UserFunctionSignature;
-import org.neo4j.kernel.api.CypherScope;
+import org.neo4j.kernel.api.QueryLanguage;
 import org.neo4j.kernel.api.ResourceMonitor;
 import org.neo4j.kernel.api.procedure.CallableProcedure;
 import org.neo4j.kernel.api.procedure.CallableUserFunction;
@@ -243,7 +243,7 @@ private long getLastUpdate() {
     public boolean registerProcedure(ProcedureSignature signature, String statement) {
         QualifiedName name = signature.name();
         try {
-            boolean exists = globalProceduresRegistry.getCurrentView().getAllProcedures(CypherScope.CYPHER_5)
+            boolean exists = globalProceduresRegistry.getCurrentView().getAllProcedures(QueryLanguage.CYPHER_5)
                     .anyMatch(s -> s.name().equals(name));
             if (exists) {
                 // we deregister and remove possible homonyms signatures overridden/overloaded
@@ -293,7 +293,7 @@ public boolean registerFunction(UserFunctionSignature signature) {
     public boolean registerFunction(UserFunctionSignature signature, String statement, boolean forceSingle, boolean mapResult) {
         try {
             QualifiedName name = signature.name();
-            boolean exists = globalProceduresRegistry.getCurrentView().getAllNonAggregatingFunctions(CypherScope.CYPHER_5)
+            boolean exists = globalProceduresRegistry.getCurrentView().getAllNonAggregatingFunctions(QueryLanguage.CYPHER_5)
                     .anyMatch(s -> s.name().equals(name));
             if (exists) {
                 // we deregister and remove possible homonyms signatures overridden/overloaded