diff --git a/src/main/scala/beam/sim/BeamHelper.scala b/src/main/scala/beam/sim/BeamHelper.scala index 78fba07af0a..c80e5ec48de 100755 --- a/src/main/scala/beam/sim/BeamHelper.scala +++ b/src/main/scala/beam/sim/BeamHelper.scala @@ -659,11 +659,17 @@ trait BeamHelper extends LazyLogging { private def prepareDirectories(config: TypesafeConfig, beamConfig: BeamConfig, outputDirectory: String): Unit = { new java.io.File(outputDirectory).mkdirs - val outConf = Paths.get(outputDirectory, "beam.conf") val location = config.getString("config") - Files.copy(Paths.get(location), outConf, StandardCopyOption.REPLACE_EXISTING) - logger.info("Config [{}] copied to {}.", beamConfig.beam.agentsim.simulationName, outConf) + val confNameToPath = BeamConfigUtils.getFileNameToPath(location) + + logger.info("Processing configs for [{}] simulation.", beamConfig.beam.agentsim.simulationName) + confNameToPath.foreach { + case (fileName, filePath) => + val outFile = Paths.get(outputDirectory, fileName) + Files.copy(Paths.get(filePath), outFile, StandardCopyOption.REPLACE_EXISTING) + logger.info("Config '{}' copied to '{}'.", filePath, outFile) + } } private def buildMatsimConfig( diff --git a/src/main/scala/beam/utils/BeamConfigUtils.scala b/src/main/scala/beam/utils/BeamConfigUtils.scala index 55a4b82719d..f2d85222e5c 100755 --- a/src/main/scala/beam/utils/BeamConfigUtils.scala +++ b/src/main/scala/beam/utils/BeamConfigUtils.scala @@ -4,8 +4,10 @@ import java.io.File import java.nio.file.Paths import com.typesafe.config.ConfigFactory +import org.apache.commons.io.FilenameUtils._ import scala.collection.JavaConverters._ +import scala.collection.{immutable, mutable} object BeamConfigUtils { @@ -23,4 +25,72 @@ object BeamConfigUtils { ) } + class ConfigPathsCollector(readFile: String => Array[String] = readFileLines) { + + private def getIncludedPaths(path: String): Array[String] = { + val basePath = getFullPath(path) + def correctRelativePath(relativePath: String): String = normalize(Paths.get(basePath, relativePath).toString) + + val lines = readFile(path) + val includedPaths = lines + .map(line => line.split('"').map(_.trim)) + .collect { case Array("include", path, _*) => path } + .map(correctRelativePath) + + includedPaths + } + + private def getIncludedPathsRecursively( + path: String, + processedPaths: immutable.HashSet[String] = immutable.HashSet.empty[String] + ): immutable.HashSet[String] = { + val includedPaths = getIncludedPaths(path) + val unprocessedPaths = includedPaths.collect { + case path if !processedPaths.contains(path) => path + } + + unprocessedPaths.foldLeft(processedPaths + path) { + case (accumulator, unprocessed) => getIncludedPathsRecursively(unprocessed, accumulator) + } + } + + def getFileNameToPath(configFileLocation: String): Map[String, String] = { + val confFileNames = mutable.Map.empty[String, Int] + val confFileLocationNormalized = normalize(configFileLocation) + val allIncludedPaths = getIncludedPathsRecursively(confFileLocationNormalized) + + val confNameToPaths = (allIncludedPaths - confFileLocationNormalized) + .map(_.toString) + .foldLeft(Map("beam.conf" -> confFileLocationNormalized)) { + case (fileNameToPath, confFilePath) => + val confFileName = getName(confFilePath) + val newConfFileName = confFileNames.get(confFileName) match { + case None => + confFileNames(confFileName) = 1 + confFileName + case Some(cnt) => + confFileNames(confFileName) = cnt + 1 + val fExtension = getExtension(confFileName) + removeExtension(confFileName) + s"_$cnt" + EXTENSION_SEPARATOR_STR + fExtension + } + + fileNameToPath + (newConfFileName -> confFilePath) + } + + confNameToPaths + } + } + + def getFileNameToPath(configFileLocation: String): Map[String, String] = { + val collector = new ConfigPathsCollector() + collector.getFileNameToPath(configFileLocation) + } + + private def readFileLines(filePath: String): Array[String] = { + val source = scala.io.Source.fromFile(filePath) + val lines = source.getLines.toArray + source.close() + lines + } + } diff --git a/src/test/scala/beam/utils/BeamConfigUtilsTest.scala b/src/test/scala/beam/utils/BeamConfigUtilsTest.scala new file mode 100644 index 00000000000..53661b61e2a --- /dev/null +++ b/src/test/scala/beam/utils/BeamConfigUtilsTest.scala @@ -0,0 +1,155 @@ +package beam.utils + +import java.nio.file.Paths + +import org.scalatest.{Matchers, WordSpecLike} + +import scala.collection.immutable.Map + +class BeamConfigUtilsTest extends WordSpecLike with Matchers { + + def getCollector(map: Map[String, Array[String]]): BeamConfigUtils.ConfigPathsCollector = + new BeamConfigUtils.ConfigPathsCollector((path: String) => { + map.get(path) match { + case Some(lines) => lines + case None => Array.empty[String] + } + }) + + def toPathStr(strPath: String): String = Paths.get(strPath).toString + private val mockText = Array( + "# This version, base-sf-light.conf, is configured to use a subsample of the population located in:", + "# ${beam.inputDirectory}\"/sample\"", + "##################################################################", + "# Agentsim", + "##################################################################", + "beam.agentsim.simulationName = \"sf-light-1k-xml\"", + "beam.agentsim.agentSampleSizeAsFractionOfPopulation = 1.0", + "beam.agentsim.firstIteration = 0", + "beam.agentsim.lastIteration = 20", + "beam.agentsim.thresholdForWalkingInMeters = 100", + "beam.agentsim.timeBinSize = 3600", + "beam.agentsim.startTime = \"00:00:00\"", + "beam.agentsim.endTime = \"30:00:00\"", + "beam.agentsim.schedulerParallelismWindow = 30" + ) + + "collector of included conf file paths " should { + "return only one path if there are no includes " in { + val filePath = "test/test2/ttt/file0.f" + val collector = getCollector(Map(filePath -> mockText)) + collector.getFileNameToPath(filePath) should equal( + Map("beam.conf" -> toPathStr(filePath)) + ) + } + + "return all path from includes " in { + val filesPaths = + Array("test/test2/supertest/file0", "test/test2/ttt/file1.fff", "test/test2/file2.f", "test/test2/ttt/file3.f") + .map(toPathStr) + val fileIncludedPaths = Array("../supertest/file0", "../file2.f", "file3.f") + val fileMap = Map( + filesPaths(1) -> Array.concat( + Array( + " include \"" + fileIncludedPaths(0) + "\"", + "some random text", + "include \"" + fileIncludedPaths(1) + "\" some text", + "include\"" + fileIncludedPaths(2) + "\"" + ), + mockText + ), + filesPaths(0) -> mockText, + filesPaths(2) -> mockText, + filesPaths(3) -> mockText + ) + + val collector = getCollector(fileMap) + collector.getFileNameToPath(filesPaths(1)) should equal( + Map( + "beam.conf" -> filesPaths(1), + "file0" -> filesPaths(0), + "file2.f" -> filesPaths(2), + "file3.f" -> filesPaths(3) + ) + ) + } + + "correctly name files if they have same names " in { + val filesPaths = Array( + "test/test2/supertest/file0.ff", + "test/test2/ttt/file1.fff", + "test/test2/file0.ff", + "test/test2/ttt/file0.ff" + ).map(toPathStr) + + val fileMap = Map( + filesPaths(1) -> Array.concat( + Array( + " include \"../supertest/file0.ff\"", + "some random text", + "include \"../file0.ff\" some text", + "include\"file0.ff\"" + ), + mockText + ), + filesPaths(0) -> mockText, + filesPaths(2) -> mockText, + filesPaths(3) -> mockText + ) + + val collector = getCollector(fileMap) + val f2p = collector.getFileNameToPath(filesPaths(1)) + + f2p.keys.toSet should equal(Set("file0.ff", "file0_1.ff", "beam.conf", "file0_2.ff")) + f2p.values.toSet should equal(Set(filesPaths(0), filesPaths(1), filesPaths(2), filesPaths(3))) + } + + "collect all includes recursively " in { + val filesPaths = + Array("test/test2/supertest/file0.ff", "test/test2/file2.fff", "test/test2/ttt/file3.f3f3").map(toPathStr) + val filePath = toPathStr("test/test2/ttt/file1.fff") + val fileMap = Map( + filePath -> (" include \"../supertest/file0.ff\"" +: mockText), + filesPaths(0) -> (" include \"../file2.fff\"" +: mockText), + filesPaths(1) -> (" include \"ttt/file3.f3f3\"" +: mockText), + filesPaths(2) -> mockText + ) + + val collector = getCollector(fileMap) + collector.getFileNameToPath(filePath) should equal( + Map( + "beam.conf" -> filePath, + "file0.ff" -> filesPaths(0), + "file2.fff" -> filesPaths(1), + "file3.f3f3" -> filesPaths(2) + ) + ) + } + + "ignore files if they appears more than one time in includes " in { + val filesPaths = Array( + "test/test2/supertest/file0.ff", + "test/test2/ttt/file1.fff", + "test/test2/file2.fff", + "test/test2/ttt/file3.f3f3" + ).map(toPathStr) + + val fileMap = Map( + filesPaths(1) -> Array.concat(Array(" include \"../supertest/file0.ff\"", " include \"file3.f3f3\""), mockText), + filesPaths(0) -> Array.concat(mockText, Array("include \"../file2.fff\" ", " include \"../ttt/file3.f3f3\" ")), + filesPaths(2) -> Array.concat(mockText, Array("include \"ttt/file1.fff\"", "include \"supertest/file0.ff\"")), + filesPaths(3) -> Array.concat(mockText, Array("include\"file1.fff\"", "include\"../supertest/file0.ff\"")) + ) + + val collector = getCollector(fileMap) + collector.getFileNameToPath(filesPaths(1)) should equal( + Map( + "beam.conf" -> filesPaths(1), + "file0.ff" -> filesPaths(0), + "file2.fff" -> filesPaths(2), + "file3.f3f3" -> filesPaths(3) + ) + ) + } + } +}