Skip to content

Commit

Permalink
[TOQo1n9M] apoc-hadoop dependency is conflicting (#3450) (#3454)
Browse files Browse the repository at this point in the history
* [TOQo1n9M] apoc-hadoop dependency is conflicting (#3450)

* [TOQo1n9M] changed for 5.x version

* [TOQo1n9M] update .gitmodules branch

* [TOQo1n9M] removed imports

* [TOQo1n9M] fixed compile error
  • Loading branch information
vga91 committed Apr 28, 2023
1 parent 8d702f4 commit 86e4484
Show file tree
Hide file tree
Showing 4 changed files with 201 additions and 104 deletions.
136 changes: 136 additions & 0 deletions extended/src/test/java/apoc/StartupExtendedTest.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,136 @@
package apoc;

import apoc.util.ExtendedTestContainerUtil;
import apoc.util.Neo4jContainerExtension;
import apoc.util.TestContainerUtil;
import apoc.util.TestContainerUtil.Neo4jVersion;
import org.apache.commons.io.FileUtils;
import org.junit.Test;
import org.neo4j.driver.Session;

import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.function.Consumer;
import java.util.function.Function;
import java.util.stream.Collectors;

import static apoc.util.TestContainerUtil.ApocPackage.CORE;
import static apoc.util.TestContainerUtil.ApocPackage.EXTENDED;
import static apoc.util.TestContainerUtil.createDB;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;

/*
This test is just to verify if the APOC procedures and functions are correctly deployed into a Neo4j instance without any startup issue.
*/
public class StartupExtendedTest {
private static final String APOC_HELP_QUERY = "CALL apoc.help('') YIELD core, type, name WHERE core = $core and type = $type RETURN name";
private static final List<String> EXPECTED_EXTENDED_NAMES;

static {
// retrieve every extended procedure and function via the extended.txt file
final File extendedFile = new File(TestContainerUtil.extendedDir, "src/main/resources/extended.txt");
try {
EXPECTED_EXTENDED_NAMES = FileUtils.readLines(extendedFile, StandardCharsets.UTF_8);
} catch (IOException e) {
throw new RuntimeException(e);
}
}

@Test
public void checkCoreAndFullWithExtraDependenciesJars() {
// we check that with apoc-extended, apoc-core jar and all extra-dependencies jars every procedure/function is detected
startContainerSessionWithExtraDeps((version) -> createDB(version, List.of(CORE, EXTENDED), true),
session -> {
checkCoreProcsAndFuncsExistence(session);

// all full procedures and functions are present, also the ones which require extra-deps, e.g. the apoc.export.xls.*
final List<String> actualExtNames = getNames(session, APOC_HELP_QUERY,
Map.of("core", false, "type", "function") );
final List<String> functionExtNames = getNames(session, APOC_HELP_QUERY,
Map.of("core", false, "type", "procedure") );

actualExtNames.addAll(functionExtNames);

assertEquals(sorted(EXPECTED_EXTENDED_NAMES), sorted(actualExtNames));
});
}

@Test
public void checkExtendedWithExtraDependenciesJars() {
// we check that with apoc-extended jar and all extra-dependencies jars every procedure/function is detected
startContainerSessionWithExtraDeps((version) -> createDB(version, List.of(EXTENDED), true),
session -> {
// all full procedures and functions are present, also the ones which require extra-deps, e.g. the apoc.export.xls.*
final List<String> actualExtNames = getNames(session, "SHOW PROCEDURES YIELD name WHERE name STARTS WITH 'apoc.' RETURN name");
final List<String> functionExtNames = getNames(session, "SHOW FUNCTIONS YIELD name WHERE name STARTS WITH 'apoc.' RETURN name");

actualExtNames.addAll(functionExtNames);

assertEquals(sorted(EXPECTED_EXTENDED_NAMES), sorted(actualExtNames));
});
}

@Test
public void checkCoreWithExtraDependenciesJars() {
// we check that with apoc-core jar and all extra-dependencies jars every procedure/function is detected
startContainerSessionWithExtraDeps((version) -> createDB(version, List.of(CORE), true),
this::checkCoreProcsAndFuncsExistence);
}

private void startContainerSessionWithExtraDeps(Function<Neo4jVersion, Neo4jContainerExtension> neo4jContainerCreation,
Consumer<Session> sessionConsumer) {
for (var version: Neo4jVersion.values()) {

try (final Neo4jContainerExtension neo4jContainer = neo4jContainerCreation.apply(version)) {
// add extra-deps before starting it
ExtendedTestContainerUtil.addExtraDependencies();
neo4jContainer.start();
assertTrue("Neo4j Instance should be up-and-running", neo4jContainer.isRunning());

final Session session = neo4jContainer.getSession();

sessionConsumer.accept(session);
} catch (Exception ex) {
// if Testcontainers wasn't able to retrieve the docker image we ignore the test
if (TestContainerUtil.isDockerImageAvailable(ex)) {
ex.printStackTrace();
fail("Should not have thrown exception when trying to start Neo4j: " + ex);
} else {
fail("The docker image could not be loaded. Check whether it's available locally / in the CI. Exception:" + ex);
}
}
}
}

private void checkCoreProcsAndFuncsExistence(Session session) {
final List<String> functionNames = getNames(session, APOC_HELP_QUERY,
Map.of("core", true, "type", "function") );

final List<String> procedureNames = getNames(session, APOC_HELP_QUERY,
Map.of("core", true, "type", "procedure") );

assertEquals(sorted(ApocSignatures.PROCEDURES), procedureNames);
assertEquals(sorted(ApocSignatures.FUNCTIONS), functionNames);
}

private static List<String> getNames(Session session, String query, Map<String, Object> params) {
return session.run(query, params)
.list(i -> i.get("name").asString());
}

private static List<String> getNames(Session session, String query) {
return getNames(session, query, Collections.emptyMap());
}

private List<String> sorted(List<String> signatures) {
return signatures.stream()
.sorted()
.collect(Collectors.toList());
}
}
19 changes: 19 additions & 0 deletions extended/src/test/java/apoc/util/ExtendedTestContainerUtil.java
Original file line number Diff line number Diff line change
@@ -1,11 +1,18 @@
package apoc.util;

import java.io.File;
import java.time.Duration;
import java.util.List;
import java.util.Map;
import java.util.function.Consumer;

import org.apache.commons.io.filefilter.IOFileFilter;
import org.apache.commons.io.filefilter.WildcardFileFilter;
import org.neo4j.driver.Session;

import static apoc.util.TestContainerUtil.copyFilesToPlugin;
import static apoc.util.TestContainerUtil.executeGradleTasks;

public class ExtendedTestContainerUtil
{
public static TestcontainersCausalCluster createEnterpriseCluster( List<TestContainerUtil.ApocPackage> apocPackages, int numOfCoreInstances, int numberOfReadReplica, Map<String, Object> neo4jConfig, Map<String, String> envSettings) {
Expand All @@ -19,4 +26,16 @@ public static <T> T singleResultFirstColumn(Session session, String cypher) {
public static void testCallInReadTransaction(Session session, String call, Consumer<Map<String, Object>> consumer) {
TestContainerUtil.testCallInReadTransaction(session, call, null, consumer);
}

public static void addExtraDependencies() {
File extraDepsDir = new File(TestContainerUtil.baseDir, "extra-dependencies");
// build the extra-dependencies
executeGradleTasks(extraDepsDir, "buildDependencies");

// add all extra deps to the plugin docker folder
final File directory = new File(extraDepsDir, "build/allJars");
final IOFileFilter instance = new WildcardFileFilter("*-all.jar");
copyFilesToPlugin(directory, instance, TestContainerUtil.pluginsFolder);
}

}
86 changes: 0 additions & 86 deletions test-startup/src/test/java/StartupTest.java

This file was deleted.

64 changes: 46 additions & 18 deletions test-utils/src/main/java/apoc/util/TestContainerUtil.java
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@

import com.github.dockerjava.api.exception.NotFoundException;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.filefilter.IOFileFilter;
import org.apache.commons.io.filefilter.TrueFileFilter;
import org.apache.commons.io.filefilter.WildcardFileFilter;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.gradle.tooling.BuildLauncher;
Expand Down Expand Up @@ -46,6 +48,7 @@ public enum ApocPackage {

private TestContainerUtil() {}

private static File pluginsFolder;
private static File baseDir = Paths.get("..").toFile();
private static File coreDir = new File(baseDir, "core");
private static File extendedDir = new File(baseDir, "extended");
Expand All @@ -68,8 +71,24 @@ public static Neo4jContainerExtension createEnterpriseDB(List<ApocPackage> apocP
public static Neo4jContainerExtension createCommunityDB(List<ApocPackage> apocPackages, boolean withLogging) {
return createNeo4jContainer(apocPackages, withLogging, Neo4jVersion.COMMUNITY);
}

private static void addExtraDependencies() {
final File projectRootDir = Paths.get("..").toFile();
File extraDepsDir = new File(projectRootDir, "extra-dependencies");
// build the extra-dependencies
executeGradleTasks(extraDepsDir, "buildDependencies");

// add all extra deps to the plugin docker folder
final File directory = new File(extraDepsDir, "build/allJars");
final IOFileFilter instance = TrueFileFilter.TRUE;
copyFilesToPlugin(directory, instance);
}

public static Neo4jContainerExtension createEnterpriseDB(File baseDir, boolean withLogging, Neo4jVersion version) {
return createEnterpriseDB(baseDir, withLogging, version, false);
}

private static Neo4jContainerExtension createNeo4jContainer(List<ApocPackage> apocPackages, boolean withLogging, Neo4jVersion version) {
private static Neo4jContainerExtension createNeo4jContainer(List<ApocPackage> apocPackages, boolean withLogging, Neo4jVersion version, boolean withExtraDeps) {
String dockerImage;
if (version == Neo4jVersion.ENTERPRISE) {
dockerImage = neo4jEnterpriseDockerImageVersion;
Expand All @@ -88,6 +107,7 @@ private static Neo4jContainerExtension createNeo4jContainer(List<ApocPackage> ap
File importFolder = new File("import");
importFolder.mkdirs();
// use a separate folder for mounting plugins jar - build/libs might contain other jars as well.
pluginsFolder = new File(baseDir, "build/plugins");
pluginsFolder.mkdirs();
String canonicalPath = null;

Expand All @@ -97,23 +117,20 @@ private static Neo4jContainerExtension createNeo4jContainer(List<ApocPackage> ap
e.printStackTrace();
}

for (ApocPackage apocPackage: apocPackages) {
if (apocPackage == ApocPackage.CORE) {
projectDir = coreDir;
} else {
projectDir = extendedDir;
}

executeGradleTasks(projectDir, "shadowJar");

Collection<File> files = FileUtils.listFiles(new File(projectDir, "build/libs"), new WildcardFileFilter(Arrays.asList("*-extended.jar", "*-core.jar")), null);
for (File file: files) {
try {
FileUtils.copyFileToDirectory(file, pluginsFolder);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
final File directory = new File(baseDir, "build/libs");
final IOFileFilter fileFilter = new WildcardFileFilter(Arrays.asList("*-extended.jar", "*-core.jar"));

copyFilesToPlugin(directory, fileFilter);

if (withExtraDeps) {
addExtraDependencies();
}

String canonicalPath = null;
try {
canonicalPath = importFolder.getCanonicalPath();
} catch (IOException e) {
e.printStackTrace();
}

System.out.println("neo4jDockerImageVersion = " + dockerImage);
Expand Down Expand Up @@ -156,6 +173,17 @@ private static Neo4jContainerExtension createNeo4jContainer(List<ApocPackage> ap
return neo4jContainer.withWaitForNeo4jDatabaseReady(password, version);
}

private static void copyFilesToPlugin(File directory, IOFileFilter instance) {
Collection<File> files = FileUtils.listFiles(directory, instance, null);
for (File file: files) {
try {
FileUtils.copyFileToDirectory(file, pluginsFolder);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
}

public static void executeGradleTasks(File baseDir, String... tasks) {
try (ProjectConnection connection = GradleConnector.newConnector()
.forProjectDirectory(baseDir)
Expand Down

0 comments on commit 86e4484

Please sign in to comment.