Warmstart: Read ride-hail fleet state (#1986)

* - Warmstart respects ride-hail fleet state
- `module` uses passed `beamConfig` instead of creating new instance
- We still need `typesafeConfig` for `ConfigModule`

* Fix tests

* Added warmstarted version of urbansim-100k

* Increase num of iters and write intervals

* include "urbansim-100k.conf"

src/main/scala/beam/sim/BeamHelper.scala

@@ -123,6 +123,7 @@ trait BeamHelper extends LazyLogging {
 
   def module(
     typesafeConfig: TypesafeConfig,
+    beamConfig: BeamConfig,
     scenario: Scenario,
     beamScenario: BeamScenario
   ): com.google.inject.Module =
@@ -137,7 +138,7 @@ trait BeamHelper extends LazyLogging {
 
           // Beam Inject below:
           install(new ConfigModule(typesafeConfig))
-          install(new BeamAgentModule(BeamConfig(typesafeConfig)))
+          install(new BeamAgentModule(beamConfig))
           install(new UtilsModule)
         }
       }).asJava,
@@ -148,8 +149,6 @@ trait BeamHelper extends LazyLogging {
         override def install(): Unit = {
           // This code will be executed 3 times due to this https://github.com/LBNL-UCB-STI/matsim/blob/master/matsim/src/main/java/org/matsim/core/controler/Injector.java#L99:L101
           // createMapBindingsForType is called 3 times. Be careful not to do expensive operations here
-          val beamConfig = BeamConfig(typesafeConfig)
-
           bind(classOf[BeamConfig]).toInstance(beamConfig)
           bind(classOf[BeamConfigChangesObservable]).toInstance(new BeamConfigChangesObservable(beamConfig))
           bind(classOf[PrepareForSim]).to(classOf[BeamPrepareForSim])
@@ -207,7 +206,7 @@ trait BeamHelper extends LazyLogging {
       }
     )
 
-  def loadScenario(beamConfig: BeamConfig) = {
+  def loadScenario(beamConfig: BeamConfig): BeamScenario = {
     val vehicleTypes = maybeScaleTransit(
       beamConfig,
       readBeamVehicleTypeFile(beamConfig.beam.agentsim.agents.vehicles.vehicleTypesFilePath)
@@ -250,7 +249,7 @@ trait BeamHelper extends LazyLogging {
     )
   }
 
-  def vehicleEnergy(beamConfig: BeamConfig, vehicleTypes: Map[Id[BeamVehicleType], BeamVehicleType]) = {
+  def vehicleEnergy(beamConfig: BeamConfig, vehicleTypes: Map[Id[BeamVehicleType], BeamVehicleType]): VehicleEnergy = {
     val baseFilePath = Paths.get(beamConfig.beam.agentsim.agents.vehicles.vehicleTypesFilePath).getParent
     val vehicleCsvReader = new VehicleCsvReader(beamConfig)
     val consumptionRateFilterStore =
@@ -444,7 +443,6 @@ trait BeamHelper extends LazyLogging {
   def runBeamWithConfig(config: TypesafeConfig): (MatsimConfig, String) = {
     val beamExecutionConfig = updateConfigWithWarmStart(setupBeamWithConfig(config))
     val (scenario, beamScenario) = buildBeamServicesAndScenario(
-      config,
       beamExecutionConfig.beamConfig,
       beamExecutionConfig.matsimConfig,
     )
@@ -470,7 +468,7 @@ trait BeamHelper extends LazyLogging {
     }
     logger.warn(logStart)
 
-    val injector: inject.Injector = buildInjector(config, scenario, beamScenario)
+    val injector: inject.Injector = buildInjector(config, beamExecutionConfig.beamConfig, scenario, beamScenario)
     val services = injector.getInstance(classOf[BeamServices])
 
     runBeam(
@@ -522,12 +520,13 @@ trait BeamHelper extends LazyLogging {
 
   protected def buildInjector(
     config: TypesafeConfig,
+    beamConfig: BeamConfig,
     scenario: MutableScenario,
     beamScenario: BeamScenario
   ): inject.Injector = {
     org.matsim.core.controler.Injector.createInjector(
       scenario.getConfig,
-      module(config, scenario, beamScenario)
+      module(config, beamConfig, scenario, beamScenario)
     )
   }
 
@@ -557,7 +556,6 @@ trait BeamHelper extends LazyLogging {
   }
 
   protected def buildBeamServicesAndScenario(
-    typesafeConfig: TypesafeConfig,
     beamConfig: BeamConfig,
     matsimConfig: MatsimConfig
   ): (MutableScenario, BeamScenario) = {

src/main/scala/beam/sim/BeamWarmStart.scala

@@ -251,6 +251,12 @@ object BeamWarmStart extends LazyLogging {
 
       val vehiclesCsv = instance.notCompressedLocation("Households", "vehicles.csv", true)
 
+      val rideHailFleetCsv = instance.compressedLocation("Ride-hail fleet state", "rideHailFleet.csv.gz")
+      val newRideHailInit = {
+        val updatedInitCfg = configAgents.rideHail.initialization.copy(filePath = rideHailFleetCsv, initType = "FILE")
+        configAgents.rideHail.copy(initialization = updatedInitCfg)
+      }
+
       val newConfigAgents = {
         val newPlans = {
           configAgents.plans
@@ -261,7 +267,12 @@ object BeamWarmStart extends LazyLogging {
 
         val newVehicles = configAgents.vehicles.copy(vehiclesFilePath = vehiclesCsv)
 
-        configAgents.copy(plans = newPlans, households = newHouseHolds, vehicles = newVehicles)
+        configAgents.copy(
+          plans = newPlans,
+          households = newHouseHolds,
+          vehicles = newVehicles,
+          rideHail = newRideHailInit
+        )
       }
 
       val newAgentSim = beamConfig.beam.agentsim.copy(agents = newConfigAgents)

src/test/scala/beam/agentsim/agents/GenericEventsSpec.scala

@@ -31,7 +31,7 @@ trait GenericEventsSpec extends WordSpecLike with IntegrationSpecCommon with Bea
 
     val injector = org.matsim.core.controler.Injector.createInjector(
       matsimConfig,
-      module(baseConfig, scenario, beamScenario)
+      module(baseConfig, beamConfig, scenario, beamScenario)
     )
 
     beamServices = injector.getInstance(classOf[BeamServices])

src/test/scala/beam/agentsim/agents/ridehail/AlonsoMoraPoolingAlgForRideHailSpec.scala

@@ -52,7 +52,7 @@ class AlonsoMoraPoolingAlgForRideHailSpec
   val beamExecConfig: BeamExecutionConfig = setupBeamWithConfig(system.settings.config)
   implicit lazy val beamScenario = loadScenario(beamExecConfig.beamConfig)
   lazy val scenario = buildScenarioFromMatsimConfig(beamExecConfig.matsimConfig, beamScenario)
-  lazy val injector = buildInjector(system.settings.config, scenario, beamScenario)
+  lazy val injector = buildInjector(system.settings.config, beamExecConfig.beamConfig, scenario, beamScenario)
   lazy val services = buildBeamServices(injector, scenario)
 
   describe("AlonsoMoraPoolingAlgForRideHail") {

src/test/scala/beam/agentsim/agents/ridehail/AsyncAlonsoMoraAlgForRideHailSpec.scala

@@ -52,7 +52,7 @@ class AsyncAlonsoMoraAlgForRideHailSpec
   val beamExecConfig: BeamExecutionConfig = setupBeamWithConfig(system.settings.config)
   implicit lazy val beamScenario = loadScenario(beamExecConfig.beamConfig)
   lazy val scenario = buildScenarioFromMatsimConfig(beamExecConfig.matsimConfig, beamScenario)
-  lazy val injector = buildInjector(system.settings.config, scenario, beamScenario)
+  lazy val injector = buildInjector(system.settings.config, beamExecConfig.beamConfig, scenario, beamScenario)
   lazy val services = buildBeamServices(injector, scenario)
   private val householdsFactory: HouseholdsFactoryImpl = new HouseholdsFactoryImpl()
 

src/test/scala/beam/agentsim/agents/ridehail/RideHailPassengersEventsSpec.scala

@@ -32,7 +32,7 @@ class RideHailPassengersEventsSpec extends WordSpecLike with Matchers with BeamH
 
       val injector = org.matsim.core.controler.Injector.createInjector(
         scenario.getConfig,
-        module(baseConfig, scenario, beamScenario)
+        module(baseConfig, beamConfig, scenario, beamScenario)
       )
 
       val beamServices: BeamServices =

src/test/scala/beam/agentsim/agents/ridehail/RideHailSurgePricingManagerSpec.scala

@@ -19,7 +19,7 @@ class RideHailSurgePricingManagerSpec extends WordSpecLike with Matchers with Mo
   val beamExecConfig: BeamExecutionConfig = setupBeamWithConfig(config)
   lazy val beamScenario = loadScenario(beamExecConfig.beamConfig)
   lazy val scenario = buildScenarioFromMatsimConfig(beamExecConfig.matsimConfig, beamScenario)
-  lazy val injector = buildInjector(config, scenario, beamScenario)
+  lazy val injector = buildInjector(config, beamExecConfig.beamConfig, scenario, beamScenario)
   lazy val beamServices = buildBeamServices(injector, scenario)
 
   "RideHailSurgePricingManager" must {

src/test/scala/beam/integration/CAVSpec.scala

@@ -48,7 +48,7 @@ class CAVSpec extends FlatSpec with Matchers with BeamHelper {
       scenario.getConfig,
       new AbstractModule() {
         override def install(): Unit = {
-          install(module(config, scenario, beamScenario))
+          install(module(config, beamConfig, scenario, beamScenario))
           addEventHandlerBinding().toInstance(new BasicEventHandler {
             override def handleEvent(event: Event): Unit = {
               event match {

src/test/scala/beam/integration/CarSharingSpec.scala

@@ -88,7 +88,7 @@ class CarSharingSpec extends FlatSpec with Matchers with BeamHelper {
       scenario.getConfig,
       new AbstractModule() {
         override def install(): Unit = {
-          install(module(config, scenario, beamScenario))
+          install(module(config, beamConfig, scenario, beamScenario))
           addEventHandlerBinding().toInstance(new BasicEventHandler {
             override def handleEvent(event: Event): Unit = {
               event match {
@@ -204,7 +204,7 @@ class CarSharingSpec extends FlatSpec with Matchers with BeamHelper {
       scenario.getConfig,
       new AbstractModule() {
         override def install(): Unit = {
-          install(module(config, scenario, beamScenario))
+          install(module(config, beamConfig, scenario, beamScenario))
           addEventHandlerBinding().toInstance(new BasicEventHandler {
             override def handleEvent(event: Event): Unit = {
               event match {

src/test/scala/beam/integration/DriveTransitSpec.scala

@@ -55,7 +55,7 @@ class DriveTransitSpec extends WordSpecLike with Matchers with BeamHelper {
         scenario.getConfig,
         new AbstractModule() {
           override def install(): Unit = {
-            install(module(config, scenario, beamScenario))
+            install(module(config, beamConfig, scenario, beamScenario))
             addEventHandlerBinding().toInstance(new BasicEventHandler {
               override def handleEvent(event: Event): Unit = {
                 event match {

src/test/scala/beam/integration/EventsFileSpec.scala

@@ -36,12 +36,11 @@ class EventsFileSpec extends FlatSpec with BeforeAndAfterAll with Matchers with
     val beamExecutionConfig: BeamExecutionConfig = setupBeamWithConfig(config)
 
     val (scenarioBuilt, beamScenario) = buildBeamServicesAndScenario(
-      config,
       beamExecutionConfig.beamConfig,
       beamExecutionConfig.matsimConfig
     )
     scenario = scenarioBuilt
-    val injector = buildInjector(config, scenario, beamScenario)
+    val injector = buildInjector(config, beamExecutionConfig.beamConfig, scenario, beamScenario)
     val services = buildBeamServices(injector, scenario)
 
     runBeam(services, scenario, beamScenario, scenario.getConfig.controler().getOutputDirectory)

src/test/scala/beam/integration/LCCMSpec.scala

@@ -39,7 +39,7 @@ class LCCMSpec extends FlatSpec with BeamHelper with MockitoSugar {
       scenario.getConfig,
       new AbstractModule() {
         override def install(): Unit = {
-          install(module(config, scenario, beamScenario))
+          install(module(config, beamConfig, scenario, beamScenario))
           addControlerListenerBinding().toInstance(iterationCounter)
         }
       }

src/test/scala/beam/integration/ridehail/RideHailAllocationRandomRepositioningSpec.scala

@@ -31,7 +31,7 @@ class RideHailAllocationRandomRepositioningSpec extends FlatSpec with BeamHelper
       scenario.getConfig,
       new AbstractModule() {
         override def install(): Unit = {
-          install(module(config, scenario, beamScenario))
+          install(module(config, beamConfig, scenario, beamScenario))
           addControlerListenerBinding().toInstance(iterationCounter)
         }
       }

src/test/scala/beam/integration/ridehail/RideHailReplaceAllocationSpec.scala

@@ -34,7 +34,7 @@ class RideHailReplaceAllocationSpec extends FlatSpec with BeamHelper with Mockit
       scenario.getConfig,
       new AbstractModule() {
         override def install(): Unit = {
-          install(module(config, scenario, beamScenario))
+          install(module(config, beamConfig, scenario, beamScenario))
           addControlerListenerBinding().toInstance(iterationCounter)
         }
       }

src/test/scala/beam/sflight/CaccSpec.scala

@@ -41,7 +41,7 @@ class CaccSpec extends WordSpecLike with Matchers with BeamHelper with BeforeAnd
     val scenario = ScenarioUtils.loadScenario(matsimConfig).asInstanceOf[MutableScenario]
     scenario.setNetwork(beamScenario.network)
 
-    val services = buildBeamServices(buildInjector(config, scenario, beamScenario), scenario)
+    val services = buildBeamServices(buildInjector(config, beamConfig, scenario, beamScenario), scenario)
     DefaultPopulationAdjustment(services).update(scenario)
 
     val controller = services.controler

src/test/scala/beam/sflight/SfLightRunSpec.scala

@@ -57,7 +57,7 @@ class SfLightRunSpec extends WordSpecLike with Matchers with BeamHelper with Bef
         scenario.getConfig,
         new AbstractModule() {
           override def install(): Unit = {
-            install(module(config, scenario, beamScenario))
+            install(module(config, beamConfig, scenario, beamScenario))
             addEventHandlerBinding().toInstance(new BasicEventHandler {
               override def handleEvent(event: Event): Unit = {
                 event match {

src/test/scala/beam/utils/SimRunnerForTest.scala

@@ -30,7 +30,7 @@ trait SimRunnerForTest extends BeamHelper with BeforeAndAfterAll { this: Suite =
     super.beforeAll()
     beamScenario = loadScenario(beamConfig)
     scenario = buildScenarioFromMatsimConfig(matsimConfig, beamScenario)
-    injector = buildInjector(config, scenario, beamScenario)
+    injector = buildInjector(config, beamConfig, scenario, beamScenario)
     services = new BeamServicesImpl(injector)
     services.modeChoiceCalculatorFactory = ModeChoiceCalculator(
       services.beamConfig.beam.agentsim.agents.modalBehaviors.modeChoiceClass,

test/input/sf-light/urbansim-100k-warmstart.conf

@@ -0,0 +1,5 @@
+include "urbansim-100k.conf"
+
+beam.agentsim.simulationName = "urbansim-100k-warmstart"
+beam.warmStart.enabled = true
+beam.warmStart.path = "https://s3.us-east-2.amazonaws.com/beam-outputs/output/sf-light/urbansim-100k__2019-07-16_18-09-53-30it.zip"

test/input/sf-light/urbansim-100k.conf

@@ -11,7 +11,7 @@ include "../common/matsim.conf"
 beam.agentsim.simulationName = "urbansim-100k"
 beam.agentsim.agentSampleSizeAsFractionOfPopulation = 1.0
 beam.agentsim.firstIteration = 0
-beam.agentsim.lastIteration = 10
+beam.agentsim.lastIteration = 20
 beam.agentsim.thresholdForWalkingInMeters = 100
 beam.agentsim.timeBinSize = 3600
 beam.agentsim.startTime = "00:00:00"
@@ -77,10 +77,7 @@ beam.agentsim.tuning.rideHailPrice = 1.0
 beam.physsim.inputNetworkFilePath = ${beam.inputDirectory}"/r5/physsim-network.xml"
 beam.physsim.flowCapacityFactor = 0.01
 beam.physsim.storageCapacityFactor = 1.0
-beam.physsim.writeEventsInterval = 0
-beam.physsim.writePlansInterval = 0
 beam.physsim.writeMATSimNetwork = false
-beam.physsim.linkStatsWriteInterval = 1
 beam.physsim.linkStatsBinSize = 3600
 beam.physsim.ptSampleSize = 0.03
 beam.physsim.jdeqsim.agentSimPhysSimInterfaceDebugger.enabled = false
@@ -164,12 +161,13 @@ beam.outputs.baseOutputDirectory = ${?BEAM_OUTPUT}
 beam.outputs.addTimestampToOutputDirectory = true
 
 # To keep all logging params in one place, BEAM overrides MATSim params normally in the controller config module
-beam.outputs.writePlansInterval = 1
-beam.outputs.writeEventsInterval = 1
-beam.outputs.writeLinkTraversalInterval = 0
+beam.physsim.linkStatsWriteInterval = 5
+beam.outputs.writeSkimsInterval = 5
+beam.outputs.writePlansInterval = 5
+beam.outputs.writeEventsInterval = 5
 
 # The remaining params customize how events are written to output files
-beam.outputs.events.fileOutputFormats = "csv.gz,xml" # valid options: xml(.gz) , csv(.gz), none - DEFAULT: csv.gz
+beam.outputs.events.fileOutputFormats = "csv.gz,xml.gz" # valid options: xml(.gz) , csv(.gz), none - DEFAULT: csv.gz
 
 # Events Writing Logging Levels:
 # Any event types not explicitly listed in overrideWritingLevels take on defaultWritingLevel