Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fixed docker testing #2328

Merged
merged 10 commits into from
Jun 21, 2024
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,11 @@ protected boolean supportsEnclaves() {
return true;
}

@Override
protected boolean supportsDockerOption() {
return true;
}

@Test
@Override
public void runBasicTests() {
Expand Down
4 changes: 3 additions & 1 deletion core/src/main/java/org/lflang/generator/LFGenerator.java
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,9 @@ public static FileConfig createFileConfig(
case Python -> new PyFileConfig(resource, srcGenBasePath, useHierarchicalBin);
case CPP -> new CppFileConfig(resource, srcGenBasePath, useHierarchicalBin);
case Rust -> new RustFileConfig(resource, srcGenBasePath, useHierarchicalBin);
case TS -> new TSFileConfig(resource, srcGenBasePath, useHierarchicalBin);
// null is passed to the docker argument because it is up to the TS Generator to tell the
// FileConfig whether Docker is enabled
case TS -> new TSFileConfig(resource, srcGenBasePath, useHierarchicalBin, null);
petervdonovan marked this conversation as resolved.
Show resolved Hide resolved
};
} catch (IOException e) {
throw new RuntimeException(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -128,16 +128,19 @@ public void createLauncher() {
var binPath = fileConfig.binPath;
FileUtil.createDirectoryIfDoesNotExist(binPath.toFile());
var file = binPath.resolve(fileConfig.name).toFile();

final var relPath =
FileUtil.toUnixString(fileConfig.binPath.relativize(fileConfig.getOutPath()));

var script =
"""
#!/bin/bash
set -euo pipefail
cd $(dirname "$0")
cd ..
cd "%s"
cd "%s/%s"
docker compose up
"""
.formatted(packageRoot.relativize(srcGenPath));
.formatted(relPath, packageRoot.relativize(srcGenPath));
var messageReporter = context.getErrorReporter();
try {
var writer = new BufferedWriter(new FileWriter(file));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import org.lflang.target.property.NoRuntimeValidationProperty
import org.lflang.target.property.PrintStatisticsProperty
import org.lflang.target.property.TracingProperty
import org.lflang.toDefinition
import org.lflang.toUnixString
import java.nio.file.Path

/** Abstract class for generating platform specific files and invoking the target compiler. */
Expand All @@ -24,6 +25,7 @@ abstract class CppPlatformGenerator(protected val generator: CppGenerator) {
protected val mainReactor = generator.mainDef.reactorClass.toDefinition()

open val srcGenPath: Path = generator.fileConfig.srcGenPath
protected val relativeBinDir = fileConfig.outPath.relativize(fileConfig.binPath).toUnixString()

abstract fun generatePlatformFiles()

Expand Down
30 changes: 17 additions & 13 deletions core/src/main/kotlin/org/lflang/generator/cpp/CppRos2Generator.kt
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ class CppRos2Generator(generator: CppGenerator) : CppPlatformGenerator(generator
)
val scriptPath =
if (targetConfig.get(DockerProperty.INSTANCE).enabled)
fileConfig.srcGenPath.resolve("bin").resolve(fileConfig.name)
fileConfig.srcGenPath.resolve(relativeBinDir).resolve(fileConfig.name)
else
fileConfig.binPath.resolve(fileConfig.name)
FileUtil.writeToFile(packageGenerator.generateBinScript(), scriptPath)
Expand All @@ -58,7 +58,8 @@ class CppRos2Generator(generator: CppGenerator) : CppPlatformGenerator(generator
return false
}
val colconCommand = commandFactory.createCommand(
"colcon", colconArgs(), fileConfig.outPath)
"colcon", colconArgs(), fileConfig.outPath
)
val returnCode = colconCommand?.run(context.cancelIndicator)
if (returnCode != 0 && !messageReporter.errorsOccurred) {
// If errors occurred but none were reported, then the following message is the best we can do.
Expand All @@ -70,13 +71,13 @@ class CppRos2Generator(generator: CppGenerator) : CppPlatformGenerator(generator

private fun colconArgs(): List<String> {
return listOf(
"build",
"--packages-select",
fileConfig.name,
packageGenerator.reactorCppName,
"--cmake-args",
"-DLF_REACTOR_CPP_SUFFIX=${packageGenerator.reactorCppSuffix}",
) + cmakeArgs
"build",
"--packages-select",
fileConfig.name,
packageGenerator.reactorCppName,
"--cmake-args",
"-DLF_REACTOR_CPP_SUFFIX=${packageGenerator.reactorCppSuffix}",
) + cmakeArgs
}

inner class CppDockerGenerator(context: LFGeneratorContext?) : DockerGenerator(context) {
Expand All @@ -90,13 +91,16 @@ class CppRos2Generator(generator: CppGenerator) : CppPlatformGenerator(generator

override fun generateRunForInstallingDeps(): String = ""

override fun defaultEntryPoint(): List<String> = listOf(fileConfig.outPath.relativize(fileConfig.binPath).toUnixString() + "/" + fileConfig.name)
override fun defaultEntryPoint(): List<String> =
listOf("$relativeBinDir/${fileConfig.name}")

override fun generateCopyOfExecutable(): String =
"""
${super.generateCopyOfExecutable()}
override fun generateCopyOfExecutable(): String {
val name = fileConfig.name
return """
COPY --from=builder /lingua-franca/$name/$relativeBinDir/$name ./$relativeBinDir/$name
COPY --from=builder lingua-franca/${fileConfig.name}/install install
""".trimIndent()
}

override fun defaultBuildCommands(): List<String> {
val commands = listOf(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -176,9 +176,7 @@ class CppStandaloneGenerator(generator: CppGenerator) :
"cmake",
cmakeArgs + additionalCmakeArgs + listOf(
"-DCMAKE_INSTALL_PREFIX=${outPath.toUnixString()}",
"-DCMAKE_INSTALL_BINDIR=${
if (outPath.isAbsolute) outPath.relativize(fileConfig.binPath).toUnixString() else fileConfig.binPath.fileName.toString()
}",
"-DCMAKE_INSTALL_BINDIR=$relativeBinDir",
"-S",
sourcesRoot ?: fileConfig.srcGenBasePath.toUnixString(),
"-B",
Expand All @@ -194,6 +192,7 @@ class CppStandaloneGenerator(generator: CppGenerator) :
return cmd
}


inner class StandaloneDockerGenerator(context: LFGeneratorContext?) : DockerGenerator(context) {

override fun generateCopyForSources(): String = "COPY src-gen src-gen"
Expand All @@ -210,15 +209,17 @@ class CppStandaloneGenerator(generator: CppGenerator) :
}
}

override fun defaultEntryPoint(): List<String> = listOf("./bin/" + context.fileConfig.name)
override fun defaultEntryPoint(): List<String> = listOf("$relativeBinDir/${fileConfig.name}")

override fun generateCopyOfExecutable(): String =
"""
${super.generateCopyOfExecutable()}
override fun generateCopyOfExecutable(): String {
val name = fileConfig.name
return """
COPY --from=builder /lingua-franca/$name/$relativeBinDir/$name ./$relativeBinDir/$name
COPY --from=builder /usr/local/lib /usr/local/lib
COPY --from=builder /usr/lib /usr/lib
COPY --from=builder /lingua-franca .
""".trimIndent()
}

override fun defaultBuildCommands(): List<String> {
val mkdirCommand = listOf("mkdir", "-p", "build")
Expand Down
24 changes: 17 additions & 7 deletions core/src/main/kotlin/org/lflang/generator/ts/TSFileConfig.kt
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ import java.nio.file.Path
* @author Hokeun Kim
*/
class TSFileConfig(
resource: Resource, srcGenBasePath: Path, useHierarchicalBin: Boolean
resource: Resource, srcGenBasePath: Path, useHierarchicalBin: Boolean, var docker: Boolean? = null
) : FileConfig(resource, srcGenBasePath, useHierarchicalBin) {

/**
Expand All @@ -54,13 +54,23 @@ class TSFileConfig(
FileUtil.deleteDirectory(srcGenPath)
}

fun setDocker(dockerEnabled: Boolean) {
docker = dockerEnabled
}

override fun getCommand(): LFCommand {
return LFCommand.get(
"node",
listOf(srcPkgPath.relativize(executable).toString()),
true,
srcPkgPath
)
if (docker == true) {
lhstrh marked this conversation as resolved.
Show resolved Hide resolved
return super.getCommand()
} else if (docker == false) {
return LFCommand.get(
"node",
listOf(srcPkgPath.relativize(executable).toString()),
true,
srcPkgPath
)
} else {
throw java.lang.IllegalStateException("The execute command cannot be determined because it is not known whether code generation is in Docker mode.")
}
}

override fun getExecutableExtension(): String {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@ class TSGenerator(
override fun doGenerate(resource: Resource, context: LFGeneratorContext) {
// Register the after delay transformation to be applied by GeneratorBase.
registerTransformation(DelayedConnectionTransformation(TSDelayBodyGenerator, targetTypes, resource, true, true))

fileConfig.setDocker(targetConfig.get(DockerProperty.INSTANCE).enabled)
super.doGenerate(resource, context)

instantiationGraph
Expand Down
99 changes: 4 additions & 95 deletions core/src/testFixtures/java/org/lflang/tests/TestBase.java
Original file line number Diff line number Diff line change
Expand Up @@ -6,16 +6,13 @@
import com.google.inject.Inject;
import com.google.inject.Injector;
import com.google.inject.Provider;
import java.io.BufferedWriter;
import java.io.File;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Arrays;
import java.util.Collections;
Expand Down Expand Up @@ -53,7 +50,6 @@
import org.lflang.tests.LFTest.Result;
import org.lflang.tests.TestRegistry.TestCategory;
import org.lflang.tests.Transformers.Transformer;
import org.lflang.util.FileUtil;
import org.lflang.util.LFCommand;

/**
Expand Down Expand Up @@ -568,104 +564,17 @@ public static String stackTraceToString(Throwable t) {
return sw.toString();
}

/** Bash script that is used to execute docker tests. */
private static final String DOCKER_RUN_SCRIPT =
"""
#!/bin/bash

# exit when any command fails
set -e

docker compose -f "$1" rm -f
docker compose -f "$1" up --build | tee docker_log.txt
docker compose -f "$1" down --rmi local

errors=`grep -E "exited with code [1-9]" docker_log.txt | cat`
rm docker_log.txt

if [[ $errors ]]; then
echo "===================================================================="
echo "ERROR: One or multiple containers exited with a non-zero exit code."
echo " See the log above for details. The following containers failed:"
echo $errors
exit 1
fi

exit 0
""";

/** Path to a bash script containing DOCKER_RUN_SCRIPT. */
private static Path dockerRunScript = null;

/**
* Return the path to a bash script containing DOCKER_RUN_SCRIPT.
*
* <p>If the script does not yet exist, it is created.
*/
private static synchronized Path getDockerRunScript() throws TestError {
if (dockerRunScript != null) {
return dockerRunScript;
}

try {
var file = File.createTempFile("run_docker_test", "sh");
file.deleteOnExit();
file.setExecutable(true);
var path = file.toPath();
try (BufferedWriter writer = Files.newBufferedWriter(path)) {
writer.write(DOCKER_RUN_SCRIPT);
}
dockerRunScript = path;
} catch (IOException e) {
throw new TestError("IO Error during test preparation.", Result.TEST_EXCEPTION, e);
}

return dockerRunScript;
}

/** Throws TestError if docker does not exist. Does nothing otherwise. */
private void checkDockerExists() throws TestError {
if (LFCommand.get("docker", List.of()) == null) {
throw new TestError("Executable 'docker' not found", Result.NO_EXEC_FAIL);
}
if (LFCommand.get("docker-compose", List.of()) == null) {
throw new TestError("Executable 'docker-compose' not found", Result.NO_EXEC_FAIL);
}
}

/**
* Return a ProcessBuilder used to test the docker execution.
*
* @param test The test to get the execution command for.
*/
private ProcessBuilder getDockerExecCommand(LFTest test) throws TestError {
checkDockerExists();
var srcGenPath = test.getFileConfig().getSrcGenPath();
var dockerComposeFile = FileUtil.globFilesEndsWith(srcGenPath, "docker-compose.yml").get(0);
return new ProcessBuilder(getDockerRunScript().toString(), dockerComposeFile.toString());
}

/**
* Return a preconfigured ProcessBuilder for executing the test program.
*
* @param test The test to get the execution command for.
*/
private ProcessBuilder getExecCommand(LFTest test) throws TestError {

var srcBasePath = test.getFileConfig().srcPkgPath.resolve("src");
var relativePathName = srcBasePath.relativize(test.getFileConfig().srcPath).toString();

// special case to test docker file generation
if (relativePathName.equalsIgnoreCase(TestCategory.DOCKER.getPath())
|| relativePathName.equalsIgnoreCase(TestCategory.DOCKER_FEDERATED.getPath())) {
return getDockerExecCommand(test);
} else {
LFCommand command = test.getFileConfig().getCommand();
if (command == null) {
throw new TestError("File: " + test.getFileConfig().getExecutable(), Result.NO_EXEC_FAIL);
}
return new ProcessBuilder(command.command()).directory(command.directory());
LFCommand command = test.getFileConfig().getCommand();
if (command == null) {
throw new TestError("File: " + test.getFileConfig().getExecutable(), Result.NO_EXEC_FAIL);
}
return new ProcessBuilder(command.command()).directory(command.directory());
}

/**
Expand Down
Loading