From 3e946ca0887934fb5da3efdf63e79a810b696468 Mon Sep 17 00:00:00 2001 From: Amit Kumar Gupta Date: Sun, 27 Jul 2014 03:12:46 -0400 Subject: [PATCH] Take AZ balancing into account. --- auctionrep/auction_rep.go | 77 +++++++++++++++++++------ auctiontypes/types.go | 4 ++ simulation/simulation_test.go | 105 +++++++++++++++++++++++++++++++--- 3 files changed, 159 insertions(+), 27 deletions(-) diff --git a/auctionrep/auction_rep.go b/auctionrep/auction_rep.go index 86dc3cc..f97b389 100644 --- a/auctionrep/auction_rep.go +++ b/auctionrep/auction_rep.go @@ -2,6 +2,7 @@ package auctionrep import ( "errors" + "math" "sync" "github.com/cloudfoundry-incubator/auction/auctiontypes" @@ -19,6 +20,10 @@ type InstanceScoreInfo struct { TotalResources auctiontypes.Resources NumInstancesOnRepForProcessGuid int NumInstancesDesiredForProcessGuid int + Index int + ProcessGuid string + RepAZNumber int + TotalNumAZs int } func New(repGuid string, delegate auctiontypes.AuctionRepDelegate) *AuctionRep { @@ -42,17 +47,22 @@ func (rep *AuctionRep) BidForStartAuction(startAuctionInfo auctiontypes.StartAuc rep.lock.Lock() defer rep.lock.Unlock() - repInstanceScoreInfo, err := rep.repInstanceScoreInfo(startAuctionInfo.ProcessGuid, startAuctionInfo.NumInstances) + instanceScoreInfo, err := rep.instanceScoreInfo( + startAuctionInfo.ProcessGuid, + startAuctionInfo.NumInstances, + startAuctionInfo.Index, + startAuctionInfo.NumAZs, + ) if err != nil { return 0, err } - err = rep.satisfiesConstraints(startAuctionInfo, repInstanceScoreInfo) + err = rep.satisfiesConstraints(startAuctionInfo, instanceScoreInfo) if err != nil { return 0, err } - return rep.startAuctionBid(repInstanceScoreInfo), nil + return rep.startAuctionBid(instanceScoreInfo), nil } // must lock here; the publicly visible operations should be atomic @@ -60,17 +70,22 @@ func (rep *AuctionRep) RebidThenTentativelyReserve(startAuctionInfo auctiontypes rep.lock.Lock() defer rep.lock.Unlock() - repInstanceScoreInfo, err := rep.repInstanceScoreInfo(startAuctionInfo.ProcessGuid, startAuctionInfo.NumInstances) + instanceScoreInfo, err := rep.instanceScoreInfo( + startAuctionInfo.ProcessGuid, + startAuctionInfo.NumInstances, + startAuctionInfo.Index, + startAuctionInfo.NumAZs, + ) if err != nil { return 0, err } - err = rep.satisfiesConstraints(startAuctionInfo, repInstanceScoreInfo) + err = rep.satisfiesConstraints(startAuctionInfo, instanceScoreInfo) if err != nil { return 0, err } - bid := rep.startAuctionBid(repInstanceScoreInfo) + bid := rep.startAuctionBid(instanceScoreInfo) //then reserve err = rep.delegate.Reserve(startAuctionInfo) @@ -102,7 +117,12 @@ func (rep *AuctionRep) BidForStopAuction(stopAuctionInfo auctiontypes.StopAuctio rep.lock.Lock() defer rep.lock.Unlock() - instanceScoreInfo, err := rep.repInstanceScoreInfo(stopAuctionInfo.ProcessGuid, stopAuctionInfo.NumInstances) + instanceScoreInfo, err := rep.instanceScoreInfo( + stopAuctionInfo.ProcessGuid, + stopAuctionInfo.NumInstances, + stopAuctionInfo.Index, + stopAuctionInfo.NumAZs, + ) if err != nil { return 0, nil, err } @@ -177,7 +197,7 @@ func (rep *AuctionRep) SimulatedInstances() []auctiontypes.SimulatedInstance { } // private internals -- no locks here -func (rep *AuctionRep) repInstanceScoreInfo(processGuid string, numInstances int) (InstanceScoreInfo, error) { +func (rep *AuctionRep) instanceScoreInfo(processGuid string, numInstances int, index int, totalNumAZs int) (InstanceScoreInfo, error) { remaining, err := rep.delegate.RemainingResources() if err != nil { return InstanceScoreInfo{}, err @@ -193,17 +213,23 @@ func (rep *AuctionRep) repInstanceScoreInfo(processGuid string, numInstances int return InstanceScoreInfo{}, err } + repAZNumber := rep.delegate.AZNumber() + return InstanceScoreInfo{ RemainingResources: remaining, TotalResources: total, NumInstancesOnRepForProcessGuid: nInstancesOnRep, NumInstancesDesiredForProcessGuid: numInstances, + Index: index, + ProcessGuid: processGuid, + RepAZNumber: repAZNumber, + TotalNumAZs: totalNumAZs, }, nil } // private internals -- no locks here -func (rep *AuctionRep) satisfiesConstraints(startAuctionInfo auctiontypes.StartAuctionInfo, repInstanceScoreInfo InstanceScoreInfo) error { - remaining := repInstanceScoreInfo.RemainingResources +func (rep *AuctionRep) satisfiesConstraints(startAuctionInfo auctiontypes.StartAuctionInfo, instanceScoreInfo InstanceScoreInfo) error { + remaining := instanceScoreInfo.RemainingResources hasEnoughMemory := remaining.MemoryMB >= startAuctionInfo.MemoryMB hasEnoughDisk := remaining.DiskMB >= startAuctionInfo.DiskMB hasEnoughContainers := remaining.Containers > 0 @@ -224,25 +250,38 @@ func (rep *AuctionRep) isRunningProcessIndex(instanceGuids []string) error { } // private internals -- no locks here -func (rep *AuctionRep) startAuctionBid(repInstanceScoreInfo InstanceScoreInfo) float64 { - remaining := repInstanceScoreInfo.RemainingResources - total := repInstanceScoreInfo.TotalResources +func (rep *AuctionRep) startAuctionBid(instanceScoreInfo InstanceScoreInfo) float64 { + remaining := instanceScoreInfo.RemainingResources + total := instanceScoreInfo.TotalResources fractionUsedContainers := 1.0 - float64(remaining.Containers)/float64(total.Containers) fractionUsedDisk := 1.0 - float64(remaining.DiskMB)/float64(total.DiskMB) fractionUsedMemory := 1.0 - float64(remaining.MemoryMB)/float64(total.MemoryMB) - fractionInstancesForProcessGuid := float64(repInstanceScoreInfo.NumInstancesOnRepForProcessGuid) / float64(repInstanceScoreInfo.NumInstancesDesiredForProcessGuid) + fractionInstancesForProcessGuid := float64(instanceScoreInfo.NumInstancesOnRepForProcessGuid) / float64(instanceScoreInfo.NumInstancesDesiredForProcessGuid) + azBalancingTerm := -math.Mod( + float64(instanceScoreInfo.Index+hash(instanceScoreInfo.ProcessGuid)+instanceScoreInfo.RepAZNumber), + float64(instanceScoreInfo.TotalNumAZs), + ) return (1.0/3.0)*fractionUsedContainers + (1.0/3.0)*fractionUsedDisk + (1.0/3.0)*fractionUsedMemory + - (1.0/1.0)*fractionInstancesForProcessGuid + (1.0/1.0)*fractionInstancesForProcessGuid + + (1.0/1.0)*azBalancingTerm +} + +func hash(s string) int { + h := 0 + for _, b := range []byte(s) { + h += int(b) + } + return h } // private internals -- no locks here -func (rep *AuctionRep) stopAuctionBid(repInstanceScoreInfo InstanceScoreInfo) float64 { - remaining := repInstanceScoreInfo.RemainingResources - total := repInstanceScoreInfo.TotalResources +func (rep *AuctionRep) stopAuctionBid(instanceScoreInfo InstanceScoreInfo) float64 { + remaining := instanceScoreInfo.RemainingResources + total := instanceScoreInfo.TotalResources fractionUsedContainers := 1.0 - float64(remaining.Containers)/float64(total.Containers) fractionUsedDisk := 1.0 - float64(remaining.DiskMB)/float64(total.DiskMB) @@ -251,5 +290,5 @@ func (rep *AuctionRep) stopAuctionBid(repInstanceScoreInfo InstanceScoreInfo) fl return (1.0/3.0)*fractionUsedContainers + (1.0/3.0)*fractionUsedDisk + (1.0/3.0)*fractionUsedMemory + - (1.0/1.0)*float64(repInstanceScoreInfo.NumInstancesOnRepForProcessGuid) + (1.0/1.0)*float64(instanceScoreInfo.NumInstancesOnRepForProcessGuid) } diff --git a/auctiontypes/types.go b/auctiontypes/types.go index aeb4c2d..7232964 100644 --- a/auctiontypes/types.go +++ b/auctiontypes/types.go @@ -104,6 +104,7 @@ func NewStartAuctionInfoFromLRPStartAuction(auction models.LRPStartAuction) Star MemoryMB: auction.MemoryMB, Index: auction.Index, NumInstances: auction.NumInstances, + NumAZs: auction.NumAZs, } } @@ -112,6 +113,7 @@ func NewStopAuctionInfoFromLRPStopAuction(auction models.LRPStopAuction) StopAuc ProcessGuid: auction.ProcessGuid, Index: auction.Index, NumInstances: auction.NumInstances, + NumAZs: auction.NumAZs, } } @@ -145,6 +147,7 @@ type StartAuctionInfo struct { MemoryMB int Index int NumInstances int + NumAZs int } func (info StartAuctionInfo) LRPIdentifier() models.LRPIdentifier { @@ -159,6 +162,7 @@ type StopAuctionInfo struct { ProcessGuid string Index int NumInstances int + NumAZs int } type SimulatedInstance struct { diff --git a/simulation/simulation_test.go b/simulation/simulation_test.go index 3f14dcc..7e22f42 100644 --- a/simulation/simulation_test.go +++ b/simulation/simulation_test.go @@ -97,6 +97,12 @@ var _ = Describe("Start and Stop Auctions", func() { n2INSTapps := []int{200, 1000} n4INSTapps := []int{50, 200} + nXSapps := []int{1500, 6000} + nXLapps := []int{55, 220} + + nSPFapps := []int{1500, 6000} + nHAapps := []int{55, 220} + for i := range nexec { i := i @@ -134,6 +140,39 @@ var _ = Describe("Start and Stop Auctions", func() { }) }) + Context("with a few very high-memory apps", func() { + It("should distribute evenly", func() { + instances := []models.LRPStartAuction{} + + instances = append(instances, generateUniqueLRPStartAuctions(nXSapps[i], 1, 1)...) + instances = append(instances, generateUniqueLRPStartAuctions(nXLapps[i], 15, 1)...) + + permutedInstances := make([]models.LRPStartAuction, len(instances)) + for i, index := range util.R.Perm(len(instances)) { + permutedInstances[i] = instances[index] + } + report := auctionDistributor.HoldAuctionsFor( + "Cold start with a few very high-memory apps", + nexec[i], + permutedInstances, + repGuids[:nexec[i]], + auctionrunner.DefaultStartAuctionRules, + ) + + visualization.PrintReport( + numAZs, + client, + report.AuctionResults, + repGuids[:nexec[i]], + report.AuctionDuration, + auctionrunner.DefaultStartAuctionRules, + ) + + svgReport.DrawReportCard(i, 0, report) + reports = append(reports, report) + }) + }) + Context("with variable instance requirements between apps", func() { It("should distribute evenly", func() { instances := []models.LRPStartAuction{} @@ -167,6 +206,39 @@ var _ = Describe("Start and Stop Auctions", func() { reports = append(reports, report) }) }) + + Context("with a few very high-instance apps", func() { + It("should distribute evenly", func() { + instances := []models.LRPStartAuction{} + + instances = append(instances, generateUniqueLRPStartAuctions(nSPFapps[i], 1, 1)...) + instances = append(instances, generateUniqueLRPStartAuctions(nHAapps[i], 1, 15)...) + + permutedInstances := make([]models.LRPStartAuction, len(instances)) + for i, index := range util.R.Perm(len(instances)) { + permutedInstances[i] = instances[index] + } + report := auctionDistributor.HoldAuctionsFor( + "Cold start with a few very high-instance apps", + nexec[i], + permutedInstances, + repGuids[:nexec[i]], + auctionrunner.DefaultStartAuctionRules, + ) + + visualization.PrintReport( + numAZs, + client, + report.AuctionResults, + repGuids[:nexec[i]], + report.AuctionDuration, + auctionrunner.DefaultStartAuctionRules, + ) + + svgReport.DrawReportCard(i, 0, report) + reports = append(reports, report) + }) + }) } }) @@ -174,22 +246,33 @@ var _ = Describe("Start and Stop Auctions", func() { nexec := []int{100, 100} nempty := []int{5, 1} napps := []int{500, 100} + numInstancesPerNewLRP := 3 + numInitialProcessPerNonEmptyRep := 50 for i := range nexec { i := i - - Context(fmt.Sprintf("%d Executors, %d Initially Empty, %d Process", nexec[i], nempty[i], napps[i]), func() { + description := fmt.Sprintf( + "%d Executors, %d Initially Empty, %d Initially Running %d Process, %d New %d-Instance Processes", + nexec[i], + nempty[i], + nexec[i]-nempty[i], + numInitialProcessPerNonEmptyRep, + napps[i], + numInstancesPerNewLRP, + ) + + Context(description, func() { BeforeEach(func() { for j := 0; j < nexec[i]-nempty[i]; j++ { - initialDistributions[j] = generateUniqueSingleIndexLRPsWithRedundantSimulatedInstances(50, 1) + initialDistributions[j] = generateUniqueSingleIndexLRPsWithRedundantSimulatedInstances(numInitialProcessPerNonEmptyRep, 1) } }) It("should distribute evenly", func() { - instances := generateUniqueLRPStartAuctions(napps[i], 1, 1) + instances := generateUniqueLRPStartAuctions(napps[i], 1, numInstancesPerNewLRP) report := auctionDistributor.HoldAuctionsFor( - "Imbalanced scenario (e.g. a deploy)", + description, nexec[i], instances, repGuids[:nexec[i]], @@ -219,8 +302,14 @@ var _ = Describe("Start and Stop Auctions", func() { for i := range nexec { i := i - - Context(fmt.Sprintf("%d Executors, roughly %d Initial Processes per Executor, %d New Processes", nexec[i], maxInitialProcessesPerExecutor, napps[i]), func() { + description := fmt.Sprintf( + "%d Executors, roughly %d Initial Processes per Executor, %d New Processes", + nexec[i], + maxInitialProcessesPerExecutor, + napps[i], + ) + + Context(description, func() { BeforeEach(func() { for j := 0; j < nexec[i]; j++ { numInstances := util.RandomIntIn(maxInitialProcessesPerExecutor-2, maxInitialProcessesPerExecutor) @@ -232,7 +321,7 @@ var _ = Describe("Start and Stop Auctions", func() { instances := generateLRPStartAuctionsForProcessGuid(napps[i], "red", 1) report := auctionDistributor.HoldAuctionsFor( - "The Watters demo", + description, nexec[i], instances, repGuids[:nexec[i]],