Merge pull request #2086 from LBNL-UCB-STI/inm/#2070-copy-all-include…

…d-config-files-to-output-folder-recursively

inm/#2070-copy-all-included-config-files-to-output-folder-recursively

src/main/scala/beam/sim/BeamHelper.scala

@@ -659,11 +659,17 @@ trait BeamHelper extends LazyLogging {
 
   private def prepareDirectories(config: TypesafeConfig, beamConfig: BeamConfig, outputDirectory: String): Unit = {
     new java.io.File(outputDirectory).mkdirs
-    val outConf = Paths.get(outputDirectory, "beam.conf")
     val location = config.getString("config")
 
-    Files.copy(Paths.get(location), outConf, StandardCopyOption.REPLACE_EXISTING)
-    logger.info("Config [{}] copied to {}.", beamConfig.beam.agentsim.simulationName, outConf)
+    val confNameToPath = BeamConfigUtils.getFileNameToPath(location)
+
+    logger.info("Processing configs for [{}] simulation.", beamConfig.beam.agentsim.simulationName)
+    confNameToPath.foreach {
+      case (fileName, filePath) =>
+        val outFile = Paths.get(outputDirectory, fileName)
+        Files.copy(Paths.get(filePath), outFile, StandardCopyOption.REPLACE_EXISTING)
+        logger.info("Config '{}' copied to '{}'.", filePath, outFile)
+    }
   }
 
   private def buildMatsimConfig(

src/main/scala/beam/utils/BeamConfigUtils.scala

@@ -4,8 +4,10 @@ import java.io.File
 import java.nio.file.Paths
 
 import com.typesafe.config.ConfigFactory
+import org.apache.commons.io.FilenameUtils._
 
 import scala.collection.JavaConverters._
+import scala.collection.{immutable, mutable}
 
 object BeamConfigUtils {
 
@@ -23,4 +25,72 @@ object BeamConfigUtils {
       )
   }
 
+  class ConfigPathsCollector(readFile: String => Array[String] = readFileLines) {
+
+    private def getIncludedPaths(path: String): Array[String] = {
+      val basePath = getFullPath(path)
+      def correctRelativePath(relativePath: String): String = normalize(Paths.get(basePath, relativePath).toString)
+
+      val lines = readFile(path)
+      val includedPaths = lines
+        .map(line => line.split('"').map(_.trim))
+        .collect { case Array("include", path, _*) => path }
+        .map(correctRelativePath)
+
+      includedPaths
+    }
+
+    private def getIncludedPathsRecursively(
+      path: String,
+      processedPaths: immutable.HashSet[String] = immutable.HashSet.empty[String]
+    ): immutable.HashSet[String] = {
+      val includedPaths = getIncludedPaths(path)
+      val unprocessedPaths = includedPaths.collect {
+        case path if !processedPaths.contains(path) => path
+      }
+
+      unprocessedPaths.foldLeft(processedPaths + path) {
+        case (accumulator, unprocessed) => getIncludedPathsRecursively(unprocessed, accumulator)
+      }
+    }
+
+    def getFileNameToPath(configFileLocation: String): Map[String, String] = {
+      val confFileNames = mutable.Map.empty[String, Int]
+      val confFileLocationNormalized = normalize(configFileLocation)
+      val allIncludedPaths = getIncludedPathsRecursively(confFileLocationNormalized)
+
+      val confNameToPaths = (allIncludedPaths - confFileLocationNormalized)
+        .map(_.toString)
+        .foldLeft(Map("beam.conf" -> confFileLocationNormalized)) {
+          case (fileNameToPath, confFilePath) =>
+            val confFileName = getName(confFilePath)
+            val newConfFileName = confFileNames.get(confFileName) match {
+              case None =>
+                confFileNames(confFileName) = 1
+                confFileName
+              case Some(cnt) =>
+                confFileNames(confFileName) = cnt + 1
+                val fExtension = getExtension(confFileName)
+                removeExtension(confFileName) + s"_$cnt" + EXTENSION_SEPARATOR_STR + fExtension
+            }
+
+            fileNameToPath + (newConfFileName -> confFilePath)
+        }
+
+      confNameToPaths
+    }
+  }
+
+  def getFileNameToPath(configFileLocation: String): Map[String, String] = {
+    val collector = new ConfigPathsCollector()
+    collector.getFileNameToPath(configFileLocation)
+  }
+
+  private def readFileLines(filePath: String): Array[String] = {
+    val source = scala.io.Source.fromFile(filePath)
+    val lines = source.getLines.toArray
+    source.close()
+    lines
+  }
+
 }

src/test/scala/beam/utils/BeamConfigUtilsTest.scala

@@ -0,0 +1,155 @@
+package beam.utils
+
+import java.nio.file.Paths
+
+import org.scalatest.{Matchers, WordSpecLike}
+
+import scala.collection.immutable.Map
+
+class BeamConfigUtilsTest extends WordSpecLike with Matchers {
+
+  def getCollector(map: Map[String, Array[String]]): BeamConfigUtils.ConfigPathsCollector =
+    new BeamConfigUtils.ConfigPathsCollector((path: String) => {
+      map.get(path) match {
+        case Some(lines) => lines
+        case None        => Array.empty[String]
+      }
+    })
+
+  def toPathStr(strPath: String): String = Paths.get(strPath).toString
+  private val mockText = Array(
+    "# This version, base-sf-light.conf, is configured to use a subsample of the population located in:",
+    "#  ${beam.inputDirectory}\"/sample\"",
+    "##################################################################",
+    "# Agentsim",
+    "##################################################################",
+    "beam.agentsim.simulationName = \"sf-light-1k-xml\"",
+    "beam.agentsim.agentSampleSizeAsFractionOfPopulation = 1.0",
+    "beam.agentsim.firstIteration = 0",
+    "beam.agentsim.lastIteration = 20",
+    "beam.agentsim.thresholdForWalkingInMeters = 100",
+    "beam.agentsim.timeBinSize = 3600",
+    "beam.agentsim.startTime = \"00:00:00\"",
+    "beam.agentsim.endTime = \"30:00:00\"",
+    "beam.agentsim.schedulerParallelismWindow = 30"
+  )
+
+  "collector of included conf file paths " should {
+    "return only one path if there are no includes " in {
+      val filePath = "test/test2/ttt/file0.f"
+      val collector = getCollector(Map(filePath -> mockText))
+      collector.getFileNameToPath(filePath) should equal(
+        Map("beam.conf" -> toPathStr(filePath))
+      )
+    }
+
+    "return all path from includes " in {
+      val filesPaths =
+        Array("test/test2/supertest/file0", "test/test2/ttt/file1.fff", "test/test2/file2.f", "test/test2/ttt/file3.f")
+          .map(toPathStr)
+      val fileIncludedPaths = Array("../supertest/file0", "../file2.f", "file3.f")
+      val fileMap = Map(
+        filesPaths(1) -> Array.concat(
+          Array(
+            " include \"" + fileIncludedPaths(0) + "\"",
+            "some random text",
+            "include    \"" + fileIncludedPaths(1) + "\" some text",
+            "include\"" + fileIncludedPaths(2) + "\""
+          ),
+          mockText
+        ),
+        filesPaths(0) -> mockText,
+        filesPaths(2) -> mockText,
+        filesPaths(3) -> mockText
+      )
+
+      val collector = getCollector(fileMap)
+      collector.getFileNameToPath(filesPaths(1)) should equal(
+        Map(
+          "beam.conf" -> filesPaths(1),
+          "file0"     -> filesPaths(0),
+          "file2.f"   -> filesPaths(2),
+          "file3.f"   -> filesPaths(3)
+        )
+      )
+    }
+
+    "correctly name files if they have same names " in {
+      val filesPaths = Array(
+        "test/test2/supertest/file0.ff",
+        "test/test2/ttt/file1.fff",
+        "test/test2/file0.ff",
+        "test/test2/ttt/file0.ff"
+      ).map(toPathStr)
+
+      val fileMap = Map(
+        filesPaths(1) -> Array.concat(
+          Array(
+            " include \"../supertest/file0.ff\"",
+            "some random text",
+            "include    \"../file0.ff\" some text",
+            "include\"file0.ff\""
+          ),
+          mockText
+        ),
+        filesPaths(0) -> mockText,
+        filesPaths(2) -> mockText,
+        filesPaths(3) -> mockText
+      )
+
+      val collector = getCollector(fileMap)
+      val f2p = collector.getFileNameToPath(filesPaths(1))
+
+      f2p.keys.toSet should equal(Set("file0.ff", "file0_1.ff", "beam.conf", "file0_2.ff"))
+      f2p.values.toSet should equal(Set(filesPaths(0), filesPaths(1), filesPaths(2), filesPaths(3)))
+    }
+
+    "collect all includes recursively " in {
+      val filesPaths =
+        Array("test/test2/supertest/file0.ff", "test/test2/file2.fff", "test/test2/ttt/file3.f3f3").map(toPathStr)
+      val filePath = toPathStr("test/test2/ttt/file1.fff")
+      val fileMap = Map(
+        filePath      -> (" include \"../supertest/file0.ff\"" +: mockText),
+        filesPaths(0) -> (" include \"../file2.fff\"" +: mockText),
+        filesPaths(1) -> (" include \"ttt/file3.f3f3\"" +: mockText),
+        filesPaths(2) -> mockText
+      )
+
+      val collector = getCollector(fileMap)
+      collector.getFileNameToPath(filePath) should equal(
+        Map(
+          "beam.conf"  -> filePath,
+          "file0.ff"   -> filesPaths(0),
+          "file2.fff"  -> filesPaths(1),
+          "file3.f3f3" -> filesPaths(2)
+        )
+      )
+    }
+
+    "ignore files if they appears more than one time in includes " in {
+      val filesPaths = Array(
+        "test/test2/supertest/file0.ff",
+        "test/test2/ttt/file1.fff",
+        "test/test2/file2.fff",
+        "test/test2/ttt/file3.f3f3"
+      ).map(toPathStr)
+
+      val fileMap = Map(
+        filesPaths(1) -> Array.concat(Array(" include \"../supertest/file0.ff\"", " include \"file3.f3f3\""), mockText),
+        filesPaths(0) -> Array.concat(mockText, Array("include \"../file2.fff\" ", " include \"../ttt/file3.f3f3\" ")),
+        filesPaths(2) -> Array.concat(mockText, Array("include \"ttt/file1.fff\"", "include \"supertest/file0.ff\"")),
+        filesPaths(3) -> Array.concat(mockText, Array("include\"file1.fff\"", "include\"../supertest/file0.ff\""))
+      )
+
+      val collector = getCollector(fileMap)
+      collector.getFileNameToPath(filesPaths(1)) should equal(
+        Map(
+          "beam.conf"  -> filesPaths(1),
+          "file0.ff"   -> filesPaths(0),
+          "file2.fff"  -> filesPaths(2),
+          "file3.f3f3" -> filesPaths(3)
+        )
+      )
+    }
+  }
+}