From ea5b0388e503ef3e755b472492bedb473965af78 Mon Sep 17 00:00:00 2001 From: REASY Date: Thu, 18 Jul 2019 20:49:24 +0700 Subject: [PATCH] Warmstart: Read ride-hail fleet state (#1986) * - Warmstart respects ride-hail fleet state - `module` uses passed `beamConfig` instead of creating new instance - We still need `typesafeConfig` for `ConfigModule` * Fix tests * Added warmstarted version of urbansim-100k * Increase num of iters and write intervals * include "urbansim-100k.conf" --- src/main/scala/beam/sim/BeamHelper.scala | 16 +++++++--------- src/main/scala/beam/sim/BeamWarmStart.scala | 13 ++++++++++++- .../beam/agentsim/agents/GenericEventsSpec.scala | 2 +- .../AlonsoMoraPoolingAlgForRideHailSpec.scala | 2 +- .../AsyncAlonsoMoraAlgForRideHailSpec.scala | 2 +- .../ridehail/RideHailPassengersEventsSpec.scala | 2 +- .../RideHailSurgePricingManagerSpec.scala | 2 +- src/test/scala/beam/integration/CAVSpec.scala | 2 +- .../scala/beam/integration/CarSharingSpec.scala | 4 ++-- .../beam/integration/DriveTransitSpec.scala | 2 +- .../scala/beam/integration/EventsFileSpec.scala | 3 +-- src/test/scala/beam/integration/LCCMSpec.scala | 2 +- ...deHailAllocationRandomRepositioningSpec.scala | 2 +- .../ridehail/RideHailReplaceAllocationSpec.scala | 2 +- src/test/scala/beam/sflight/CaccSpec.scala | 2 +- src/test/scala/beam/sflight/SfLightRunSpec.scala | 2 +- src/test/scala/beam/utils/SimRunnerForTest.scala | 2 +- test/input/sf-light/urbansim-100k-warmstart.conf | 5 +++++ test/input/sf-light/urbansim-100k.conf | 14 ++++++-------- 19 files changed, 46 insertions(+), 35 deletions(-) create mode 100644 test/input/sf-light/urbansim-100k-warmstart.conf diff --git a/src/main/scala/beam/sim/BeamHelper.scala b/src/main/scala/beam/sim/BeamHelper.scala index cca7cc135da..768867d6a64 100755 --- a/src/main/scala/beam/sim/BeamHelper.scala +++ b/src/main/scala/beam/sim/BeamHelper.scala @@ -123,6 +123,7 @@ trait BeamHelper extends LazyLogging { def module( typesafeConfig: TypesafeConfig, + beamConfig: BeamConfig, scenario: Scenario, beamScenario: BeamScenario ): com.google.inject.Module = @@ -137,7 +138,7 @@ trait BeamHelper extends LazyLogging { // Beam Inject below: install(new ConfigModule(typesafeConfig)) - install(new BeamAgentModule(BeamConfig(typesafeConfig))) + install(new BeamAgentModule(beamConfig)) install(new UtilsModule) } }).asJava, @@ -148,8 +149,6 @@ trait BeamHelper extends LazyLogging { override def install(): Unit = { // This code will be executed 3 times due to this https://github.com/LBNL-UCB-STI/matsim/blob/master/matsim/src/main/java/org/matsim/core/controler/Injector.java#L99:L101 // createMapBindingsForType is called 3 times. Be careful not to do expensive operations here - val beamConfig = BeamConfig(typesafeConfig) - bind(classOf[BeamConfig]).toInstance(beamConfig) bind(classOf[BeamConfigChangesObservable]).toInstance(new BeamConfigChangesObservable(beamConfig)) bind(classOf[PrepareForSim]).to(classOf[BeamPrepareForSim]) @@ -207,7 +206,7 @@ trait BeamHelper extends LazyLogging { } ) - def loadScenario(beamConfig: BeamConfig) = { + def loadScenario(beamConfig: BeamConfig): BeamScenario = { val vehicleTypes = maybeScaleTransit( beamConfig, readBeamVehicleTypeFile(beamConfig.beam.agentsim.agents.vehicles.vehicleTypesFilePath) @@ -250,7 +249,7 @@ trait BeamHelper extends LazyLogging { ) } - def vehicleEnergy(beamConfig: BeamConfig, vehicleTypes: Map[Id[BeamVehicleType], BeamVehicleType]) = { + def vehicleEnergy(beamConfig: BeamConfig, vehicleTypes: Map[Id[BeamVehicleType], BeamVehicleType]): VehicleEnergy = { val baseFilePath = Paths.get(beamConfig.beam.agentsim.agents.vehicles.vehicleTypesFilePath).getParent val vehicleCsvReader = new VehicleCsvReader(beamConfig) val consumptionRateFilterStore = @@ -444,7 +443,6 @@ trait BeamHelper extends LazyLogging { def runBeamWithConfig(config: TypesafeConfig): (MatsimConfig, String) = { val beamExecutionConfig = updateConfigWithWarmStart(setupBeamWithConfig(config)) val (scenario, beamScenario) = buildBeamServicesAndScenario( - config, beamExecutionConfig.beamConfig, beamExecutionConfig.matsimConfig, ) @@ -470,7 +468,7 @@ trait BeamHelper extends LazyLogging { } logger.warn(logStart) - val injector: inject.Injector = buildInjector(config, scenario, beamScenario) + val injector: inject.Injector = buildInjector(config, beamExecutionConfig.beamConfig, scenario, beamScenario) val services = injector.getInstance(classOf[BeamServices]) runBeam( @@ -522,12 +520,13 @@ trait BeamHelper extends LazyLogging { protected def buildInjector( config: TypesafeConfig, + beamConfig: BeamConfig, scenario: MutableScenario, beamScenario: BeamScenario ): inject.Injector = { org.matsim.core.controler.Injector.createInjector( scenario.getConfig, - module(config, scenario, beamScenario) + module(config, beamConfig, scenario, beamScenario) ) } @@ -557,7 +556,6 @@ trait BeamHelper extends LazyLogging { } protected def buildBeamServicesAndScenario( - typesafeConfig: TypesafeConfig, beamConfig: BeamConfig, matsimConfig: MatsimConfig ): (MutableScenario, BeamScenario) = { diff --git a/src/main/scala/beam/sim/BeamWarmStart.scala b/src/main/scala/beam/sim/BeamWarmStart.scala index e4f25320707..6fbb1b726ec 100755 --- a/src/main/scala/beam/sim/BeamWarmStart.scala +++ b/src/main/scala/beam/sim/BeamWarmStart.scala @@ -251,6 +251,12 @@ object BeamWarmStart extends LazyLogging { val vehiclesCsv = instance.notCompressedLocation("Households", "vehicles.csv", true) + val rideHailFleetCsv = instance.compressedLocation("Ride-hail fleet state", "rideHailFleet.csv.gz") + val newRideHailInit = { + val updatedInitCfg = configAgents.rideHail.initialization.copy(filePath = rideHailFleetCsv, initType = "FILE") + configAgents.rideHail.copy(initialization = updatedInitCfg) + } + val newConfigAgents = { val newPlans = { configAgents.plans @@ -261,7 +267,12 @@ object BeamWarmStart extends LazyLogging { val newVehicles = configAgents.vehicles.copy(vehiclesFilePath = vehiclesCsv) - configAgents.copy(plans = newPlans, households = newHouseHolds, vehicles = newVehicles) + configAgents.copy( + plans = newPlans, + households = newHouseHolds, + vehicles = newVehicles, + rideHail = newRideHailInit + ) } val newAgentSim = beamConfig.beam.agentsim.copy(agents = newConfigAgents) diff --git a/src/test/scala/beam/agentsim/agents/GenericEventsSpec.scala b/src/test/scala/beam/agentsim/agents/GenericEventsSpec.scala index e8e3e387958..b647a6590f0 100755 --- a/src/test/scala/beam/agentsim/agents/GenericEventsSpec.scala +++ b/src/test/scala/beam/agentsim/agents/GenericEventsSpec.scala @@ -31,7 +31,7 @@ trait GenericEventsSpec extends WordSpecLike with IntegrationSpecCommon with Bea val injector = org.matsim.core.controler.Injector.createInjector( matsimConfig, - module(baseConfig, scenario, beamScenario) + module(baseConfig, beamConfig, scenario, beamScenario) ) beamServices = injector.getInstance(classOf[BeamServices]) diff --git a/src/test/scala/beam/agentsim/agents/ridehail/AlonsoMoraPoolingAlgForRideHailSpec.scala b/src/test/scala/beam/agentsim/agents/ridehail/AlonsoMoraPoolingAlgForRideHailSpec.scala index e093f5bbaf0..683bf854af9 100644 --- a/src/test/scala/beam/agentsim/agents/ridehail/AlonsoMoraPoolingAlgForRideHailSpec.scala +++ b/src/test/scala/beam/agentsim/agents/ridehail/AlonsoMoraPoolingAlgForRideHailSpec.scala @@ -52,7 +52,7 @@ class AlonsoMoraPoolingAlgForRideHailSpec val beamExecConfig: BeamExecutionConfig = setupBeamWithConfig(system.settings.config) implicit lazy val beamScenario = loadScenario(beamExecConfig.beamConfig) lazy val scenario = buildScenarioFromMatsimConfig(beamExecConfig.matsimConfig, beamScenario) - lazy val injector = buildInjector(system.settings.config, scenario, beamScenario) + lazy val injector = buildInjector(system.settings.config, beamExecConfig.beamConfig, scenario, beamScenario) lazy val services = buildBeamServices(injector, scenario) describe("AlonsoMoraPoolingAlgForRideHail") { diff --git a/src/test/scala/beam/agentsim/agents/ridehail/AsyncAlonsoMoraAlgForRideHailSpec.scala b/src/test/scala/beam/agentsim/agents/ridehail/AsyncAlonsoMoraAlgForRideHailSpec.scala index 99dd117f47d..15d602376ce 100644 --- a/src/test/scala/beam/agentsim/agents/ridehail/AsyncAlonsoMoraAlgForRideHailSpec.scala +++ b/src/test/scala/beam/agentsim/agents/ridehail/AsyncAlonsoMoraAlgForRideHailSpec.scala @@ -52,7 +52,7 @@ class AsyncAlonsoMoraAlgForRideHailSpec val beamExecConfig: BeamExecutionConfig = setupBeamWithConfig(system.settings.config) implicit lazy val beamScenario = loadScenario(beamExecConfig.beamConfig) lazy val scenario = buildScenarioFromMatsimConfig(beamExecConfig.matsimConfig, beamScenario) - lazy val injector = buildInjector(system.settings.config, scenario, beamScenario) + lazy val injector = buildInjector(system.settings.config, beamExecConfig.beamConfig, scenario, beamScenario) lazy val services = buildBeamServices(injector, scenario) private val householdsFactory: HouseholdsFactoryImpl = new HouseholdsFactoryImpl() diff --git a/src/test/scala/beam/agentsim/agents/ridehail/RideHailPassengersEventsSpec.scala b/src/test/scala/beam/agentsim/agents/ridehail/RideHailPassengersEventsSpec.scala index 8086e9483b6..3acd37c9e8d 100755 --- a/src/test/scala/beam/agentsim/agents/ridehail/RideHailPassengersEventsSpec.scala +++ b/src/test/scala/beam/agentsim/agents/ridehail/RideHailPassengersEventsSpec.scala @@ -32,7 +32,7 @@ class RideHailPassengersEventsSpec extends WordSpecLike with Matchers with BeamH val injector = org.matsim.core.controler.Injector.createInjector( scenario.getConfig, - module(baseConfig, scenario, beamScenario) + module(baseConfig, beamConfig, scenario, beamScenario) ) val beamServices: BeamServices = diff --git a/src/test/scala/beam/agentsim/agents/ridehail/RideHailSurgePricingManagerSpec.scala b/src/test/scala/beam/agentsim/agents/ridehail/RideHailSurgePricingManagerSpec.scala index daf40ddf401..894e088ca01 100755 --- a/src/test/scala/beam/agentsim/agents/ridehail/RideHailSurgePricingManagerSpec.scala +++ b/src/test/scala/beam/agentsim/agents/ridehail/RideHailSurgePricingManagerSpec.scala @@ -19,7 +19,7 @@ class RideHailSurgePricingManagerSpec extends WordSpecLike with Matchers with Mo val beamExecConfig: BeamExecutionConfig = setupBeamWithConfig(config) lazy val beamScenario = loadScenario(beamExecConfig.beamConfig) lazy val scenario = buildScenarioFromMatsimConfig(beamExecConfig.matsimConfig, beamScenario) - lazy val injector = buildInjector(config, scenario, beamScenario) + lazy val injector = buildInjector(config, beamExecConfig.beamConfig, scenario, beamScenario) lazy val beamServices = buildBeamServices(injector, scenario) "RideHailSurgePricingManager" must { diff --git a/src/test/scala/beam/integration/CAVSpec.scala b/src/test/scala/beam/integration/CAVSpec.scala index 809ccd5f790..5f4d8141335 100644 --- a/src/test/scala/beam/integration/CAVSpec.scala +++ b/src/test/scala/beam/integration/CAVSpec.scala @@ -48,7 +48,7 @@ class CAVSpec extends FlatSpec with Matchers with BeamHelper { scenario.getConfig, new AbstractModule() { override def install(): Unit = { - install(module(config, scenario, beamScenario)) + install(module(config, beamConfig, scenario, beamScenario)) addEventHandlerBinding().toInstance(new BasicEventHandler { override def handleEvent(event: Event): Unit = { event match { diff --git a/src/test/scala/beam/integration/CarSharingSpec.scala b/src/test/scala/beam/integration/CarSharingSpec.scala index 97e6495eebd..d974f6c8387 100644 --- a/src/test/scala/beam/integration/CarSharingSpec.scala +++ b/src/test/scala/beam/integration/CarSharingSpec.scala @@ -88,7 +88,7 @@ class CarSharingSpec extends FlatSpec with Matchers with BeamHelper { scenario.getConfig, new AbstractModule() { override def install(): Unit = { - install(module(config, scenario, beamScenario)) + install(module(config, beamConfig, scenario, beamScenario)) addEventHandlerBinding().toInstance(new BasicEventHandler { override def handleEvent(event: Event): Unit = { event match { @@ -204,7 +204,7 @@ class CarSharingSpec extends FlatSpec with Matchers with BeamHelper { scenario.getConfig, new AbstractModule() { override def install(): Unit = { - install(module(config, scenario, beamScenario)) + install(module(config, beamConfig, scenario, beamScenario)) addEventHandlerBinding().toInstance(new BasicEventHandler { override def handleEvent(event: Event): Unit = { event match { diff --git a/src/test/scala/beam/integration/DriveTransitSpec.scala b/src/test/scala/beam/integration/DriveTransitSpec.scala index b5eff7a6058..f2deba3fd87 100755 --- a/src/test/scala/beam/integration/DriveTransitSpec.scala +++ b/src/test/scala/beam/integration/DriveTransitSpec.scala @@ -55,7 +55,7 @@ class DriveTransitSpec extends WordSpecLike with Matchers with BeamHelper { scenario.getConfig, new AbstractModule() { override def install(): Unit = { - install(module(config, scenario, beamScenario)) + install(module(config, beamConfig, scenario, beamScenario)) addEventHandlerBinding().toInstance(new BasicEventHandler { override def handleEvent(event: Event): Unit = { event match { diff --git a/src/test/scala/beam/integration/EventsFileSpec.scala b/src/test/scala/beam/integration/EventsFileSpec.scala index b7ab6fd1352..d31f7c56d92 100755 --- a/src/test/scala/beam/integration/EventsFileSpec.scala +++ b/src/test/scala/beam/integration/EventsFileSpec.scala @@ -36,12 +36,11 @@ class EventsFileSpec extends FlatSpec with BeforeAndAfterAll with Matchers with val beamExecutionConfig: BeamExecutionConfig = setupBeamWithConfig(config) val (scenarioBuilt, beamScenario) = buildBeamServicesAndScenario( - config, beamExecutionConfig.beamConfig, beamExecutionConfig.matsimConfig ) scenario = scenarioBuilt - val injector = buildInjector(config, scenario, beamScenario) + val injector = buildInjector(config, beamExecutionConfig.beamConfig, scenario, beamScenario) val services = buildBeamServices(injector, scenario) runBeam(services, scenario, beamScenario, scenario.getConfig.controler().getOutputDirectory) diff --git a/src/test/scala/beam/integration/LCCMSpec.scala b/src/test/scala/beam/integration/LCCMSpec.scala index fbdf085756d..695abf9d1fe 100755 --- a/src/test/scala/beam/integration/LCCMSpec.scala +++ b/src/test/scala/beam/integration/LCCMSpec.scala @@ -39,7 +39,7 @@ class LCCMSpec extends FlatSpec with BeamHelper with MockitoSugar { scenario.getConfig, new AbstractModule() { override def install(): Unit = { - install(module(config, scenario, beamScenario)) + install(module(config, beamConfig, scenario, beamScenario)) addControlerListenerBinding().toInstance(iterationCounter) } } diff --git a/src/test/scala/beam/integration/ridehail/RideHailAllocationRandomRepositioningSpec.scala b/src/test/scala/beam/integration/ridehail/RideHailAllocationRandomRepositioningSpec.scala index 2e5bba48d70..d066e8764f9 100755 --- a/src/test/scala/beam/integration/ridehail/RideHailAllocationRandomRepositioningSpec.scala +++ b/src/test/scala/beam/integration/ridehail/RideHailAllocationRandomRepositioningSpec.scala @@ -31,7 +31,7 @@ class RideHailAllocationRandomRepositioningSpec extends FlatSpec with BeamHelper scenario.getConfig, new AbstractModule() { override def install(): Unit = { - install(module(config, scenario, beamScenario)) + install(module(config, beamConfig, scenario, beamScenario)) addControlerListenerBinding().toInstance(iterationCounter) } } diff --git a/src/test/scala/beam/integration/ridehail/RideHailReplaceAllocationSpec.scala b/src/test/scala/beam/integration/ridehail/RideHailReplaceAllocationSpec.scala index 224a80e5d89..30e94854e05 100755 --- a/src/test/scala/beam/integration/ridehail/RideHailReplaceAllocationSpec.scala +++ b/src/test/scala/beam/integration/ridehail/RideHailReplaceAllocationSpec.scala @@ -34,7 +34,7 @@ class RideHailReplaceAllocationSpec extends FlatSpec with BeamHelper with Mockit scenario.getConfig, new AbstractModule() { override def install(): Unit = { - install(module(config, scenario, beamScenario)) + install(module(config, beamConfig, scenario, beamScenario)) addControlerListenerBinding().toInstance(iterationCounter) } } diff --git a/src/test/scala/beam/sflight/CaccSpec.scala b/src/test/scala/beam/sflight/CaccSpec.scala index 55eed8509b1..8d7233f7f01 100644 --- a/src/test/scala/beam/sflight/CaccSpec.scala +++ b/src/test/scala/beam/sflight/CaccSpec.scala @@ -41,7 +41,7 @@ class CaccSpec extends WordSpecLike with Matchers with BeamHelper with BeforeAnd val scenario = ScenarioUtils.loadScenario(matsimConfig).asInstanceOf[MutableScenario] scenario.setNetwork(beamScenario.network) - val services = buildBeamServices(buildInjector(config, scenario, beamScenario), scenario) + val services = buildBeamServices(buildInjector(config, beamConfig, scenario, beamScenario), scenario) DefaultPopulationAdjustment(services).update(scenario) val controller = services.controler diff --git a/src/test/scala/beam/sflight/SfLightRunSpec.scala b/src/test/scala/beam/sflight/SfLightRunSpec.scala index c5f0063091c..c3ba9a80baa 100755 --- a/src/test/scala/beam/sflight/SfLightRunSpec.scala +++ b/src/test/scala/beam/sflight/SfLightRunSpec.scala @@ -57,7 +57,7 @@ class SfLightRunSpec extends WordSpecLike with Matchers with BeamHelper with Bef scenario.getConfig, new AbstractModule() { override def install(): Unit = { - install(module(config, scenario, beamScenario)) + install(module(config, beamConfig, scenario, beamScenario)) addEventHandlerBinding().toInstance(new BasicEventHandler { override def handleEvent(event: Event): Unit = { event match { diff --git a/src/test/scala/beam/utils/SimRunnerForTest.scala b/src/test/scala/beam/utils/SimRunnerForTest.scala index 0cf229fbf4b..0ad69fad953 100644 --- a/src/test/scala/beam/utils/SimRunnerForTest.scala +++ b/src/test/scala/beam/utils/SimRunnerForTest.scala @@ -30,7 +30,7 @@ trait SimRunnerForTest extends BeamHelper with BeforeAndAfterAll { this: Suite = super.beforeAll() beamScenario = loadScenario(beamConfig) scenario = buildScenarioFromMatsimConfig(matsimConfig, beamScenario) - injector = buildInjector(config, scenario, beamScenario) + injector = buildInjector(config, beamConfig, scenario, beamScenario) services = new BeamServicesImpl(injector) services.modeChoiceCalculatorFactory = ModeChoiceCalculator( services.beamConfig.beam.agentsim.agents.modalBehaviors.modeChoiceClass, diff --git a/test/input/sf-light/urbansim-100k-warmstart.conf b/test/input/sf-light/urbansim-100k-warmstart.conf new file mode 100644 index 00000000000..f44e7f96ac7 --- /dev/null +++ b/test/input/sf-light/urbansim-100k-warmstart.conf @@ -0,0 +1,5 @@ +include "urbansim-100k.conf" + +beam.agentsim.simulationName = "urbansim-100k-warmstart" +beam.warmStart.enabled = true +beam.warmStart.path = "https://s3.us-east-2.amazonaws.com/beam-outputs/output/sf-light/urbansim-100k__2019-07-16_18-09-53-30it.zip" diff --git a/test/input/sf-light/urbansim-100k.conf b/test/input/sf-light/urbansim-100k.conf index 2489d4197e8..7ff1b59ced2 100755 --- a/test/input/sf-light/urbansim-100k.conf +++ b/test/input/sf-light/urbansim-100k.conf @@ -11,7 +11,7 @@ include "../common/matsim.conf" beam.agentsim.simulationName = "urbansim-100k" beam.agentsim.agentSampleSizeAsFractionOfPopulation = 1.0 beam.agentsim.firstIteration = 0 -beam.agentsim.lastIteration = 10 +beam.agentsim.lastIteration = 20 beam.agentsim.thresholdForWalkingInMeters = 100 beam.agentsim.timeBinSize = 3600 beam.agentsim.startTime = "00:00:00" @@ -77,10 +77,7 @@ beam.agentsim.tuning.rideHailPrice = 1.0 beam.physsim.inputNetworkFilePath = ${beam.inputDirectory}"/r5/physsim-network.xml" beam.physsim.flowCapacityFactor = 0.01 beam.physsim.storageCapacityFactor = 1.0 -beam.physsim.writeEventsInterval = 0 -beam.physsim.writePlansInterval = 0 beam.physsim.writeMATSimNetwork = false -beam.physsim.linkStatsWriteInterval = 1 beam.physsim.linkStatsBinSize = 3600 beam.physsim.ptSampleSize = 0.03 beam.physsim.jdeqsim.agentSimPhysSimInterfaceDebugger.enabled = false @@ -164,12 +161,13 @@ beam.outputs.baseOutputDirectory = ${?BEAM_OUTPUT} beam.outputs.addTimestampToOutputDirectory = true # To keep all logging params in one place, BEAM overrides MATSim params normally in the controller config module -beam.outputs.writePlansInterval = 1 -beam.outputs.writeEventsInterval = 1 -beam.outputs.writeLinkTraversalInterval = 0 +beam.physsim.linkStatsWriteInterval = 5 +beam.outputs.writeSkimsInterval = 5 +beam.outputs.writePlansInterval = 5 +beam.outputs.writeEventsInterval = 5 # The remaining params customize how events are written to output files -beam.outputs.events.fileOutputFormats = "csv.gz,xml" # valid options: xml(.gz) , csv(.gz), none - DEFAULT: csv.gz +beam.outputs.events.fileOutputFormats = "csv.gz,xml.gz" # valid options: xml(.gz) , csv(.gz), none - DEFAULT: csv.gz # Events Writing Logging Levels: # Any event types not explicitly listed in overrideWritingLevels take on defaultWritingLevel