Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[TOQo1n9M] apoc-hadoop dependency is conflicting #3450

Merged
merged 1 commit into from
Feb 16, 2023
Merged

[TOQo1n9M] apoc-hadoop dependency is conflicting #3450

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions extra-dependencies/hadoop/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,8 @@ shadowJar {
def commonExclusions = {
exclude group: 'io.netty'
exclude group: 'com.sun.jersey'
exclude group: 'org.slf4j'
exclude group: 'org.apache.commons', module: 'commons-compress'
}

dependencies {
Expand Down
102 changes: 71 additions & 31 deletions test-startup/src/test/java/StartupTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,17 @@
import apoc.util.Neo4jContainerExtension;
import apoc.util.TestContainerUtil;
import apoc.util.TestUtil;
import org.apache.commons.io.FileUtils;
import org.junit.Test;
import org.neo4j.driver.Session;

import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Paths;
import java.util.List;
import java.util.function.Consumer;
import java.util.function.Supplier;
import java.util.stream.Collectors;

import static apoc.util.TestContainerUtil.createEnterpriseDB;
Expand All @@ -22,64 +27,99 @@
public class StartupTest {

private static final File APOC_FULL;
private static final File APOC_CORE;

private static File getProjectPath(String basePath, String project) {
return Paths.get( basePath.concat(project) )
.toFile();
Comment on lines +33 to +34
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Comparing with the old code, there was a slash in the concat method: .concat("/full") now this will be .concat(full). Does that matter?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, I forgot to mention it.

How it was before wasn't totally right,
because basePath is for example /My/Path/neo4j-apoc-procedures/.

By doing basePath.concat("/full") becomes /My/Path/neo4j-apoc-procedures//full (with a double slash before full).

Then Paths.get(...) removes the double slash indeed,
but I guess it's better to remove it beforehand.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ok that makes sense, thanks for explaining

}

static {
final String file = StartupTest.class.getClassLoader().getResource(".").getFile();
final int endIndex = file.indexOf("test-startup");
APOC_FULL = Paths.get(file.substring(0, endIndex).concat("/full")).toFile();
final String basePath = file.substring(0, endIndex);
APOC_FULL = getProjectPath(basePath, "full");
APOC_CORE = getProjectPath(basePath, "core");
}

@Test
public void check_basic_deployment() {
try (Neo4jContainerExtension neo4jContainer = createEnterpriseDB(APOC_FULL, !TestUtil.isRunningInCI())
.withNeo4jConfig("dbms.transaction.timeout", "5s")) {
startNeo4jContainerSession(() -> createEnterpriseDB(APOC_FULL, !TestUtil.isRunningInCI())
.withNeo4jConfig("dbms.transaction.timeout", "5s"),
session -> {
int procedureCount = session.run("CALL dbms.procedures() YIELD name WHERE name STARTS WITH 'apoc' RETURN count(*) AS count").peek().get("count").asInt();
int functionCount = session.run("CALL dbms.functions() YIELD name WHERE name STARTS WITH 'apoc' RETURN count(*) AS count").peek().get("count").asInt();
int coreCount = session.run("CALL apoc.help('') YIELD core WHERE core = true RETURN count(*) AS count").peek().get("count").asInt();

neo4jContainer.start();
assertTrue("Neo4j Instance should be up-and-running", neo4jContainer.isRunning());
assertTrue(procedureCount > 0);
assertTrue(functionCount > 0);
assertTrue(coreCount > 0);
});
}

Session session = neo4jContainer.getSession();
int procedureCount = session.run("CALL dbms.procedures() YIELD name WHERE name STARTS WITH 'apoc' RETURN count(*) AS count").peek().get("count").asInt();
int functionCount = session.run("CALL dbms.functions() YIELD name WHERE name STARTS WITH 'apoc' RETURN count(*) AS count").peek().get("count").asInt();
int coreCount = session.run("CALL apoc.help('') YIELD core WHERE core = true RETURN count(*) AS count").peek().get("count").asInt();
@Test
public void compare_with_sources() {
startNeo4jContainerSession(() -> createEnterpriseDB(APOC_FULL, !TestUtil.isRunningInCI()),
this::checkCoreProcsAndFuncsExistence);

assertTrue(procedureCount > 0);
assertTrue(functionCount > 0);
assertTrue(coreCount > 0);
} catch (Exception ex) {
// if Testcontainers wasn't able to retrieve the docker image we ignore the test
if (TestContainerUtil.isDockerImageAvailable(ex)) {
ex.printStackTrace();
fail("Should not have thrown exception when trying to start Neo4j: " + ex);
}
}
}

@Test
public void compare_with_sources() {
try (Neo4jContainerExtension neo4jContainer = createEnterpriseDB(APOC_FULL, !TestUtil.isRunningInCI())) {
neo4jContainer.start();
public void checkFullWithExtraDependenciesJars() throws IOException {

assertTrue("Neo4j Instance should be up-and-running", neo4jContainer.isRunning());
// we retrieve every full procedure and function via the extended.txt file
final File extendedFile = new File(APOC_FULL, "src/main/resources/extended.txt");
final List<String> expectedFullProcAndFunNames = FileUtils.readLines(extendedFile, StandardCharsets.UTF_8);

// we check that with apoc-full jar and all extra-dependencies jars every procedure/function is detected
startNeo4jContainerSession(() -> createEnterpriseDB(APOC_FULL, !TestUtil.isRunningInCI(), true),
session -> {
checkCoreProcsAndFuncsExistence(session);

try (Session session = neo4jContainer.getSession()) {
final List<String> functionNames = session.run("CALL apoc.help('') YIELD core, type, name WHERE core = true and type = 'function' RETURN name")
.list(record -> record.get("name").asString());
final List<String> procedureNames = session.run("CALL apoc.help('') YIELD core, type, name WHERE core = true and type = 'procedure' RETURN name")
.list(record -> record.get("name").asString());
// all full procedures and functions are present, also the ones which require extra-deps, e.g. the apoc.export.xls.*
final List<String> actualFullProcAndFunNames = session.run("CALL apoc.help('') YIELD core, type, name WHERE core = false RETURN name").list(i -> i.get("name").asString());
assertEquals(sorted(expectedFullProcAndFunNames), sorted(actualFullProcAndFunNames));
});
}

@Test
public void checkCoreWithExtraDependenciesJars() {
// we check that with apoc-core jar and all extra-dependencies jars every procedure/function is detected
startNeo4jContainerSession(() -> createEnterpriseDB(APOC_CORE, !TestUtil.isRunningInCI(), true),
this::checkCoreProcsAndFuncsExistence);

assertEquals(sorted(ApocSignatures.PROCEDURES), procedureNames);
assertEquals(sorted(ApocSignatures.FUNCTIONS), functionNames);
}
}

private void startNeo4jContainerSession(Supplier<Neo4jContainerExtension> neo4jContainerCreation,
Consumer<Session> sessionConsumer) {
try (final Neo4jContainerExtension neo4jContainer = neo4jContainerCreation.get()) {
neo4jContainer.start();
assertTrue("Neo4j Instance should be up-and-running", neo4jContainer.isRunning());

final Session session = neo4jContainer.getSession();

sessionConsumer.accept(session);
} catch (Exception ex) {
// if Testcontainers wasn't able to retrieve the docker image we ignore the test
if (TestContainerUtil.isDockerImageAvailable(ex)) {
ex.printStackTrace();
fail("Should not have thrown exception when trying to start Neo4j: " + ex);
} else {
fail( "The docker image could not be loaded. Check whether it's available locally / in the CI. Exception:" + ex);
}
}
}

private void checkCoreProcsAndFuncsExistence(Session session) {
final List<String> functionNames = session.run("CALL apoc.help('') YIELD core, type, name WHERE core = true and type = 'function' RETURN name")
.list(record -> record.get("name").asString());
final List<String> procedureNames = session.run("CALL apoc.help('') YIELD core, type, name WHERE core = true and type = 'procedure' RETURN name")
.list(record -> record.get("name").asString());

assertEquals(sorted(ApocSignatures.PROCEDURES), procedureNames);
assertEquals(sorted(ApocSignatures.FUNCTIONS), functionNames);
}

private List<String> sorted(List<String> signatures) {
return signatures.stream().sorted().collect(Collectors.toList());
}
Expand Down
47 changes: 39 additions & 8 deletions test-utils/src/main/java/apoc/util/TestContainerUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@

import com.github.dockerjava.api.exception.NotFoundException;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.filefilter.IOFileFilter;
import org.apache.commons.io.filefilter.TrueFileFilter;
import org.apache.commons.io.filefilter.WildcardFileFilter;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.gradle.tooling.BuildLauncher;
Expand Down Expand Up @@ -31,6 +33,7 @@ public class TestContainerUtil {

private TestContainerUtil() {}

private static File pluginsFolder;
private static File baseDir = Paths.get(".").toFile();

public static TestcontainersCausalCluster createEnterpriseCluster(int numOfCoreInstances, int numberOfReadReplica, Map<String, Object> neo4jConfig, Map<String, String> envSettings) {
Expand All @@ -40,8 +43,24 @@ public static TestcontainersCausalCluster createEnterpriseCluster(int numOfCoreI
public static Neo4jContainerExtension createEnterpriseDB(boolean withLogging) {
return createEnterpriseDB(baseDir, withLogging);
}

private static void addExtraDependencies() {
final File projectRootDir = Paths.get("..").toFile();
File extraDepsDir = new File(projectRootDir, "extra-dependencies");
// build the extra-dependencies
executeGradleTasks(extraDepsDir, "buildDependencies");

// add all extra deps to the plugin docker folder
final File directory = new File(extraDepsDir, "build/allJars");
final IOFileFilter instance = TrueFileFilter.TRUE;
copyFilesToPlugin(directory, instance);
}

public static Neo4jContainerExtension createEnterpriseDB(File baseDir, boolean withLogging) {
return createEnterpriseDB(baseDir, withLogging, false);
}

public static Neo4jContainerExtension createEnterpriseDB(File baseDir, boolean withLogging, boolean withExtraDeps) {
executeGradleTasks(baseDir, "shadowJar");
// We define the container with external volumes
File importFolder = new File("import");
Expand All @@ -51,17 +70,18 @@ public static Neo4jContainerExtension createEnterpriseDB(File baseDir, boolean w
String neo4jDockerImageVersion = System.getProperty("neo4jDockerImage", "neo4j:4.4.16-enterprise");

// use a separate folder for mounting plugins jar - build/libs might contain other jars as well.
File pluginsFolder = new File(baseDir, "build/plugins");
pluginsFolder = new File(baseDir, "build/plugins");
pluginsFolder.mkdirs();

Collection<File> files = FileUtils.listFiles(new File(baseDir, "build/libs"), new WildcardFileFilter(Arrays.asList("*-all.jar", "*-core.jar")), null);
for (File file: files) {
try {
FileUtils.copyFileToDirectory(file, pluginsFolder);
} catch (IOException e) {
throw new RuntimeException(e);
}
final File directory = new File(baseDir, "build/libs");
final IOFileFilter fileFilter = new WildcardFileFilter(Arrays.asList("*-all.jar", "*-core.jar"));

copyFilesToPlugin(directory, fileFilter);

if (withExtraDeps) {
addExtraDependencies();
}

String canonicalPath = null;
try {
canonicalPath = importFolder.getCanonicalPath();
Expand Down Expand Up @@ -104,6 +124,17 @@ public static Neo4jContainerExtension createEnterpriseDB(File baseDir, boolean w
return neo4jContainer;
}

private static void copyFilesToPlugin(File directory, IOFileFilter instance) {
Collection<File> files = FileUtils.listFiles(directory, instance, null);
for (File file: files) {
try {
FileUtils.copyFileToDirectory(file, pluginsFolder);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
}

public static void executeGradleTasks(File baseDir, String... tasks) {
try (ProjectConnection connection = GradleConnector.newConnector()
.forProjectDirectory(baseDir)
Expand Down