Skip to content

Commit

Permalink
Take AZ balancing into account.
Browse files Browse the repository at this point in the history
  • Loading branch information
@amitkgupta
amitkgupta committed Jul 27, 2014
1 parent 8e9ec8d commit 3e946ca
Show file tree
Hide file tree
Showing 3 changed files with 159 additions and 27 deletions.
77 changes: 58 additions & 19 deletions auctionrep/auction_rep.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package auctionrep

import (
"errors"
"math"
"sync"

"github.com/cloudfoundry-incubator/auction/auctiontypes"
Expand All @@ -19,6 +20,10 @@ type InstanceScoreInfo struct {
TotalResources auctiontypes.Resources
NumInstancesOnRepForProcessGuid int
NumInstancesDesiredForProcessGuid int
Index int
ProcessGuid string
RepAZNumber int
TotalNumAZs int
}

func New(repGuid string, delegate auctiontypes.AuctionRepDelegate) *AuctionRep {
Expand All @@ -42,35 +47,45 @@ func (rep *AuctionRep) BidForStartAuction(startAuctionInfo auctiontypes.StartAuc
rep.lock.Lock()
defer rep.lock.Unlock()

repInstanceScoreInfo, err := rep.repInstanceScoreInfo(startAuctionInfo.ProcessGuid, startAuctionInfo.NumInstances)
instanceScoreInfo, err := rep.instanceScoreInfo(
startAuctionInfo.ProcessGuid,
startAuctionInfo.NumInstances,
startAuctionInfo.Index,
startAuctionInfo.NumAZs,
)
if err != nil {
return 0, err
}

err = rep.satisfiesConstraints(startAuctionInfo, repInstanceScoreInfo)
err = rep.satisfiesConstraints(startAuctionInfo, instanceScoreInfo)
if err != nil {
return 0, err
}

return rep.startAuctionBid(repInstanceScoreInfo), nil
return rep.startAuctionBid(instanceScoreInfo), nil
}

// must lock here; the publicly visible operations should be atomic
func (rep *AuctionRep) RebidThenTentativelyReserve(startAuctionInfo auctiontypes.StartAuctionInfo) (float64, error) {
rep.lock.Lock()
defer rep.lock.Unlock()

repInstanceScoreInfo, err := rep.repInstanceScoreInfo(startAuctionInfo.ProcessGuid, startAuctionInfo.NumInstances)
instanceScoreInfo, err := rep.instanceScoreInfo(
startAuctionInfo.ProcessGuid,
startAuctionInfo.NumInstances,
startAuctionInfo.Index,
startAuctionInfo.NumAZs,
)
if err != nil {
return 0, err
}

err = rep.satisfiesConstraints(startAuctionInfo, repInstanceScoreInfo)
err = rep.satisfiesConstraints(startAuctionInfo, instanceScoreInfo)
if err != nil {
return 0, err
}

bid := rep.startAuctionBid(repInstanceScoreInfo)
bid := rep.startAuctionBid(instanceScoreInfo)

//then reserve
err = rep.delegate.Reserve(startAuctionInfo)
Expand Down Expand Up @@ -102,7 +117,12 @@ func (rep *AuctionRep) BidForStopAuction(stopAuctionInfo auctiontypes.StopAuctio
rep.lock.Lock()
defer rep.lock.Unlock()

instanceScoreInfo, err := rep.repInstanceScoreInfo(stopAuctionInfo.ProcessGuid, stopAuctionInfo.NumInstances)
instanceScoreInfo, err := rep.instanceScoreInfo(
stopAuctionInfo.ProcessGuid,
stopAuctionInfo.NumInstances,
stopAuctionInfo.Index,
stopAuctionInfo.NumAZs,
)
if err != nil {
return 0, nil, err
}
Expand Down Expand Up @@ -177,7 +197,7 @@ func (rep *AuctionRep) SimulatedInstances() []auctiontypes.SimulatedInstance {
}

// private internals -- no locks here
func (rep *AuctionRep) repInstanceScoreInfo(processGuid string, numInstances int) (InstanceScoreInfo, error) {
func (rep *AuctionRep) instanceScoreInfo(processGuid string, numInstances int, index int, totalNumAZs int) (InstanceScoreInfo, error) {
remaining, err := rep.delegate.RemainingResources()
if err != nil {
return InstanceScoreInfo{}, err
Expand All @@ -193,17 +213,23 @@ func (rep *AuctionRep) repInstanceScoreInfo(processGuid string, numInstances int
return InstanceScoreInfo{}, err
}

repAZNumber := rep.delegate.AZNumber()

return InstanceScoreInfo{
RemainingResources: remaining,
TotalResources: total,
NumInstancesOnRepForProcessGuid: nInstancesOnRep,
NumInstancesDesiredForProcessGuid: numInstances,
Index: index,
ProcessGuid: processGuid,
RepAZNumber: repAZNumber,
TotalNumAZs: totalNumAZs,
}, nil
}

// private internals -- no locks here
func (rep *AuctionRep) satisfiesConstraints(startAuctionInfo auctiontypes.StartAuctionInfo, repInstanceScoreInfo InstanceScoreInfo) error {
remaining := repInstanceScoreInfo.RemainingResources
func (rep *AuctionRep) satisfiesConstraints(startAuctionInfo auctiontypes.StartAuctionInfo, instanceScoreInfo InstanceScoreInfo) error {
remaining := instanceScoreInfo.RemainingResources
hasEnoughMemory := remaining.MemoryMB >= startAuctionInfo.MemoryMB
hasEnoughDisk := remaining.DiskMB >= startAuctionInfo.DiskMB
hasEnoughContainers := remaining.Containers > 0
Expand All @@ -224,25 +250,38 @@ func (rep *AuctionRep) isRunningProcessIndex(instanceGuids []string) error {
}

// private internals -- no locks here
func (rep *AuctionRep) startAuctionBid(repInstanceScoreInfo InstanceScoreInfo) float64 {
remaining := repInstanceScoreInfo.RemainingResources
total := repInstanceScoreInfo.TotalResources
func (rep *AuctionRep) startAuctionBid(instanceScoreInfo InstanceScoreInfo) float64 {
remaining := instanceScoreInfo.RemainingResources
total := instanceScoreInfo.TotalResources

fractionUsedContainers := 1.0 - float64(remaining.Containers)/float64(total.Containers)
fractionUsedDisk := 1.0 - float64(remaining.DiskMB)/float64(total.DiskMB)
fractionUsedMemory := 1.0 - float64(remaining.MemoryMB)/float64(total.MemoryMB)
fractionInstancesForProcessGuid := float64(repInstanceScoreInfo.NumInstancesOnRepForProcessGuid) / float64(repInstanceScoreInfo.NumInstancesDesiredForProcessGuid)
fractionInstancesForProcessGuid := float64(instanceScoreInfo.NumInstancesOnRepForProcessGuid) / float64(instanceScoreInfo.NumInstancesDesiredForProcessGuid)
azBalancingTerm := -math.Mod(
float64(instanceScoreInfo.Index+hash(instanceScoreInfo.ProcessGuid)+instanceScoreInfo.RepAZNumber),
float64(instanceScoreInfo.TotalNumAZs),
)

return (1.0/3.0)*fractionUsedContainers +
(1.0/3.0)*fractionUsedDisk +
(1.0/3.0)*fractionUsedMemory +
(1.0/1.0)*fractionInstancesForProcessGuid
(1.0/1.0)*fractionInstancesForProcessGuid +
(1.0/1.0)*azBalancingTerm
}

func hash(s string) int {
h := 0
for _, b := range []byte(s) {
h += int(b)
}
return h
}

// private internals -- no locks here
func (rep *AuctionRep) stopAuctionBid(repInstanceScoreInfo InstanceScoreInfo) float64 {
remaining := repInstanceScoreInfo.RemainingResources
total := repInstanceScoreInfo.TotalResources
func (rep *AuctionRep) stopAuctionBid(instanceScoreInfo InstanceScoreInfo) float64 {
remaining := instanceScoreInfo.RemainingResources
total := instanceScoreInfo.TotalResources

fractionUsedContainers := 1.0 - float64(remaining.Containers)/float64(total.Containers)
fractionUsedDisk := 1.0 - float64(remaining.DiskMB)/float64(total.DiskMB)
Expand All @@ -251,5 +290,5 @@ func (rep *AuctionRep) stopAuctionBid(repInstanceScoreInfo InstanceScoreInfo) fl
return (1.0/3.0)*fractionUsedContainers +
(1.0/3.0)*fractionUsedDisk +
(1.0/3.0)*fractionUsedMemory +
(1.0/1.0)*float64(repInstanceScoreInfo.NumInstancesOnRepForProcessGuid)
(1.0/1.0)*float64(instanceScoreInfo.NumInstancesOnRepForProcessGuid)
}
4 changes: 4 additions & 0 deletions auctiontypes/types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,7 @@ func NewStartAuctionInfoFromLRPStartAuction(auction models.LRPStartAuction) Star
MemoryMB: auction.MemoryMB,
Index: auction.Index,
NumInstances: auction.NumInstances,
NumAZs: auction.NumAZs,
}
}

Expand All @@ -112,6 +113,7 @@ func NewStopAuctionInfoFromLRPStopAuction(auction models.LRPStopAuction) StopAuc
ProcessGuid: auction.ProcessGuid,
Index: auction.Index,
NumInstances: auction.NumInstances,
NumAZs: auction.NumAZs,
}
}

Expand Down Expand Up @@ -145,6 +147,7 @@ type StartAuctionInfo struct {
MemoryMB int
Index int
NumInstances int
NumAZs int
}

func (info StartAuctionInfo) LRPIdentifier() models.LRPIdentifier {
Expand All @@ -159,6 +162,7 @@ type StopAuctionInfo struct {
ProcessGuid string
Index int
NumInstances int
NumAZs int
}

type SimulatedInstance struct {
Expand Down
105 changes: 97 additions & 8 deletions simulation/simulation_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,12 @@ var _ = Describe("Start and Stop Auctions", func() {
n2INSTapps := []int{200, 1000}
n4INSTapps := []int{50, 200}

nXSapps := []int{1500, 6000}
nXLapps := []int{55, 220}

nSPFapps := []int{1500, 6000}
nHAapps := []int{55, 220}

for i := range nexec {
i := i

Expand Down Expand Up @@ -134,6 +140,39 @@ var _ = Describe("Start and Stop Auctions", func() {
})
})

Context("with a few very high-memory apps", func() {
It("should distribute evenly", func() {
instances := []models.LRPStartAuction{}

instances = append(instances, generateUniqueLRPStartAuctions(nXSapps[i], 1, 1)...)
instances = append(instances, generateUniqueLRPStartAuctions(nXLapps[i], 15, 1)...)

permutedInstances := make([]models.LRPStartAuction, len(instances))
for i, index := range util.R.Perm(len(instances)) {
permutedInstances[i] = instances[index]
}
report := auctionDistributor.HoldAuctionsFor(
"Cold start with a few very high-memory apps",
nexec[i],
permutedInstances,
repGuids[:nexec[i]],
auctionrunner.DefaultStartAuctionRules,
)

visualization.PrintReport(
numAZs,
client,
report.AuctionResults,
repGuids[:nexec[i]],
report.AuctionDuration,
auctionrunner.DefaultStartAuctionRules,
)

svgReport.DrawReportCard(i, 0, report)
reports = append(reports, report)
})
})

Context("with variable instance requirements between apps", func() {
It("should distribute evenly", func() {
instances := []models.LRPStartAuction{}
Expand Down Expand Up @@ -167,29 +206,73 @@ var _ = Describe("Start and Stop Auctions", func() {
reports = append(reports, report)
})
})

Context("with a few very high-instance apps", func() {
It("should distribute evenly", func() {
instances := []models.LRPStartAuction{}

instances = append(instances, generateUniqueLRPStartAuctions(nSPFapps[i], 1, 1)...)
instances = append(instances, generateUniqueLRPStartAuctions(nHAapps[i], 1, 15)...)

permutedInstances := make([]models.LRPStartAuction, len(instances))
for i, index := range util.R.Perm(len(instances)) {
permutedInstances[i] = instances[index]
}
report := auctionDistributor.HoldAuctionsFor(
"Cold start with a few very high-instance apps",
nexec[i],
permutedInstances,
repGuids[:nexec[i]],
auctionrunner.DefaultStartAuctionRules,
)

visualization.PrintReport(
numAZs,
client,
report.AuctionResults,
repGuids[:nexec[i]],
report.AuctionDuration,
auctionrunner.DefaultStartAuctionRules,
)

svgReport.DrawReportCard(i, 0, report)
reports = append(reports, report)
})
})
}
})

Context("With some executors empty, and the remaining executors running many perfectly-balanced process instances", func() {
nexec := []int{100, 100}
nempty := []int{5, 1}
napps := []int{500, 100}
numInstancesPerNewLRP := 3
numInitialProcessPerNonEmptyRep := 50

for i := range nexec {
i := i

Context(fmt.Sprintf("%d Executors, %d Initially Empty, %d Process", nexec[i], nempty[i], napps[i]), func() {
description := fmt.Sprintf(
"%d Executors, %d Initially Empty, %d Initially Running %d Process, %d New %d-Instance Processes",
nexec[i],
nempty[i],
nexec[i]-nempty[i],
numInitialProcessPerNonEmptyRep,
napps[i],
numInstancesPerNewLRP,
)

Context(description, func() {
BeforeEach(func() {
for j := 0; j < nexec[i]-nempty[i]; j++ {
initialDistributions[j] = generateUniqueSingleIndexLRPsWithRedundantSimulatedInstances(50, 1)
initialDistributions[j] = generateUniqueSingleIndexLRPsWithRedundantSimulatedInstances(numInitialProcessPerNonEmptyRep, 1)
}
})

It("should distribute evenly", func() {
instances := generateUniqueLRPStartAuctions(napps[i], 1, 1)
instances := generateUniqueLRPStartAuctions(napps[i], 1, numInstancesPerNewLRP)

report := auctionDistributor.HoldAuctionsFor(
"Imbalanced scenario (e.g. a deploy)",
description,
nexec[i],
instances,
repGuids[:nexec[i]],
Expand Down Expand Up @@ -219,8 +302,14 @@ var _ = Describe("Start and Stop Auctions", func() {

for i := range nexec {
i := i

Context(fmt.Sprintf("%d Executors, roughly %d Initial Processes per Executor, %d New Processes", nexec[i], maxInitialProcessesPerExecutor, napps[i]), func() {
description := fmt.Sprintf(
"%d Executors, roughly %d Initial Processes per Executor, %d New Processes",
nexec[i],
maxInitialProcessesPerExecutor,
napps[i],
)

Context(description, func() {
BeforeEach(func() {
for j := 0; j < nexec[i]; j++ {
numInstances := util.RandomIntIn(maxInitialProcessesPerExecutor-2, maxInitialProcessesPerExecutor)
Expand All @@ -232,7 +321,7 @@ var _ = Describe("Start and Stop Auctions", func() {
instances := generateLRPStartAuctionsForProcessGuid(napps[i], "red", 1)

report := auctionDistributor.HoldAuctionsFor(
"The Watters demo",
description,
nexec[i],
instances,
repGuids[:nexec[i]],
Expand Down

0 comments on commit 3e946ca

Please sign in to comment.