Merge pull request #1880 from LBNL-UCB-STI/rajnikantsh/#1862-check-du…

…plicate-config-key-4ci

check duplicate keys

src/main/scala/beam/utils/ConfigConsistencyComparator.scala

@@ -7,6 +7,10 @@ import scala.collection.JavaConverters._
 import com.typesafe.config.{ConfigException, ConfigFactory, ConfigResolveOptions, ConfigValue, Config => TypesafeConfig}
 import com.typesafe.scalalogging.LazyLogging
 
+import scala.collection.mutable
+import scala.io.Source
+import scala.util.Try
+
 object ConfigConsistencyComparator extends LazyLogging {
   private val eol = System.lineSeparator()
   private val borderLeft = "**  "
@@ -35,6 +39,12 @@ object ConfigConsistencyComparator extends LazyLogging {
     val userConf = userBeamConf.withFallback(userMatsimConf).resolve(configResolver)
     val templateConf = ConfigFactory.parseFile(new File("src/main/resources/beam-template.conf")).resolve()
 
+    val duplicateKeys = findDuplicateKeys(userConfFileLocation)
+    if (duplicateKeys.nonEmpty) {
+      val title = "Found the following duplicate config keys from your config file:"
+      logStringBuilder.append(buildTopicWithKeys(title, duplicateKeys))
+    }
+
     val deprecatedKeys = findDeprecatedKeys(userConf, templateConf)
     if (deprecatedKeys.nonEmpty) {
       val title = "Found the following deprecated parameters, you can safely remove them from your config file:"
@@ -65,6 +75,26 @@ object ConfigConsistencyComparator extends LazyLogging {
     }
   }
 
+  //This method filter duplicate only for non nested keys
+  def findDuplicateKeys(userConfFileLocation: String): Seq[String] = {
+
+    val lines = Try(Source.fromFile(userConfFileLocation).getLines().toList).getOrElse(List())
+    val bracketStack = mutable.Stack[String]()
+    val configKey = mutable.Map[String, Int]().withDefaultValue(0)
+    val withoutCommentConfigLines = lines.withFilter(!_.trim.startsWith("#"))
+    for (line <- withoutCommentConfigLines) {
+      if (line.contains("{") && !line.contains("${")) {
+        bracketStack.push("{")
+      } else if (line.contains("}") && !line.contains("${")) {
+        bracketStack.pop()
+      } else if (bracketStack.isEmpty && line.contains("=")) {
+        val keyedValue = line.split("=")
+        configKey.update(keyedValue(0).trim, configKey(keyedValue(0).trim) + 1)
+      }
+    }
+    configKey.retain((_, value) => value > 1).keys.toSeq
+  }
+
   def findDeprecatedKeys(userConf: TypesafeConfig, templateConf: TypesafeConfig): Seq[String] = {
     userConf
       .entrySet()

src/test/scala/beam/utils/DuplicateConfigKeySpec.scala

@@ -0,0 +1,53 @@
+package beam.utils
+import java.io.{File, PrintWriter}
+
+import org.scalatest.{Matchers, WordSpecLike}
+
+class DuplicateConfigKeySpec extends WordSpecLike with Matchers {
+
+  val dummyConfString =
+    """
+      |beam.agentsim.simulationName = "beamville"
+      |beam.agentsim.agentSampleSizeAsFractionOfPopulation = 0.1
+      |beam.agentsim.numAgents = 100
+      |beam.agentsim.firstIteration = 0
+      |beam.agentsim.lastIteration = 2
+      |beam.agentsim.simulationName = "sf-light-1k"
+      |beam.agentsim.agentSampleSizeAsFractionOfPopulation = 0.2
+      |beam.agentsim.simulationName = "sf-light-2k"
+      |###########################
+      |# Replanning
+      |###########################
+      |beam.replanning{
+      |  maxAgentPlanMemorySize = 4
+      |  Module_1 = "SelectExpBeta"
+      |  ModuleProbability_1 = 0.7
+      |}
+      |
+    """.stripMargin
+
+  "Duplicate Config Finder" must {
+    "count duplicate config keys" in {
+      val file = File.createTempFile("somePrefix", ".conf")
+      try {
+        writeToFile(file, dummyConfString)
+        val duplicateKeys = ConfigConsistencyComparator.findDuplicateKeys(file.getPath)
+        duplicateKeys should contain("beam.agentsim.simulationName")
+        duplicateKeys should contain("beam.agentsim.agentSampleSizeAsFractionOfPopulation")
+        duplicateKeys should not contain "beam.agentsim.numAgents"
+        duplicateKeys.size should be(2)
+      } finally {
+        file.delete()
+      }
+    }
+  }
+
+  private def writeToFile(file: File, content: String): Unit = {
+    val writer = new PrintWriter(file)
+    try {
+      writer.println(content)
+    } finally {
+      writer.close()
+    }
+  }
+}