From 92648dab4953a649f0f0e3f6e9a0ece7f9bfaf78 Mon Sep 17 00:00:00 2001 From: Chris Hager Date: Sun, 9 Apr 2023 16:43:46 +0200 Subject: [PATCH] support multiple parents (#340) * support multiple parents * ensure withdrawals check in capella * readme updates * cleanup & pr review --- README.md | 27 ++-- services/api/service.go | 283 ++++++++++++---------------------------- services/api/utils.go | 4 + 3 files changed, 100 insertions(+), 214 deletions(-) diff --git a/README.md b/README.md index 6f4c3fde..da6469bf 100644 --- a/README.md +++ b/README.md @@ -36,15 +36,23 @@ The relay consists of several components that are designed to run and scale inde 1. Redis 1. PostgreSQL -1. one or more [beacon nodes](#running-beacon-node--s-) (note: run multiple beacon nodes!) +1. one or more beacon nodes 1. block submission validation nodes 1. [optional] Memcached -#### About beacon nodes: +#### Beacon nodes / CL clients -* Relays are strongly advised to run multiple beacon nodes localy! -* The reason being that on getPayload, the block has to be accepted by a local beacon node before it is returned to the proposer. -* If the local beacon nodes don't accept it, the block won't be returned to the proposer, which leads to the proposer missing the slot. +- The relay services need access to one or more beacon node for event subscriptions (in particular the `head` and `payload_attributes` topics). +- You can specify multiple beacon nodes by providing a comma separated list of beacon node URIs. +- The beacon nodes need to support the []`payload_attributes` SSE event](https://github.com/ethereum/beacon-APIs/pull/305). +- As of now, this is either: + - **Lighthouse v4.0.1+** (with `--always-prepare-payload` and `--prepare-payload-lookahead 12000` flags, and some junk feeRecipeint), with the [validate-before-broadcast patch](https://github.com/sigp/lighthouse/pull/4168). Here's a [quick guide](https://gist.github.com/metachris/bcae9ae42e2fc834804241f991351c4e) for setting up Lighthouse. + - **Prysm v4.0.0+** with the [validate-before-broadcast patch](https://github.com/flashbots/prysm/pull/17/commits/11f997f5933654cfd6e2c8298b61cd1d38bb6d5d) or the more experimental [fast-validate-before-broadcast patch](https://gist.github.com/terencechain/8dbd40da7a640b4833fbedf0976595ad) + +**Relays are strongly advised to run multiple beacon nodes!** +* The reason is that on getPayload, the block has to be validated and broadcast by a local beacon node before it is returned to the proposer. +* If the local beacon nodes don't accept it (i.e. because it's down), the block won't be returned to the proposer, which leads to the proposer missing the slot. +* The relay makes the validate+broadcast request to all beacon nodes concurrently, and returns as soon as the first request is successful. #### Security @@ -82,14 +90,6 @@ Read more in [Why run mev-boost?](https://writings.flashbots.net/writings/why-ru # Usage -## Running Beacon Node(s) - -- The services need access to a beacon node for event subscriptions. You can specify multiple beacon nodes by providing a comma separated list of beacon node URIs. - - The default beacon API is `localhost:3500` (Prysm default beacon-API port) -- The beacon node needs to support the `payload_attributes` SSE event [[1]](https://github.com/ethereum/beacon-APIs/pull/305). As of now, this is either: - - Prysm v4.0.0+ - - Lighthouse v4.0.1+ (with `--always-prepare-payload` and `--prepare-payload-lookahead 12000` flags, and some junk feeRecipeint) - ## Running Postgres, Redis and Memcached ```bash # Start PostgreSQL & Redis individually: @@ -153,7 +153,6 @@ redis-cli DEL boost-relay/sepolia:validators-registration boost-relay/sepolia:va * `DISABLE_PAYLOAD_DATABASE_STORAGE` - builder API - disable storing execution payloads in the database (i.e. when using memcached as data availability redundancy) * `DISABLE_LOWPRIO_BUILDERS` - reject block submissions by low-prio builders * `FORCE_GET_HEADER_204` - force 204 as getHeader response -* `DISABLE_SSE_PAYLOAD_ATTRIBUTES` - instead of using SSE events, poll withdrawals and randao (requires custom Prysm fork) * `MEMCACHE_ALLOW_SAVING_FAIL` -- don't abort builder submission if memcache saving fails #### Development Environment Variables diff --git a/services/api/service.go b/services/api/service.go index abb55046..0d496edb 100644 --- a/services/api/service.go +++ b/services/api/service.go @@ -110,14 +110,11 @@ type RelayAPIOpts struct { InternalAPI bool } -type randaoHelper struct { - slot uint64 - prevRandao string -} - -type withdrawalsHelper struct { - slot uint64 - root phase0.Root +type payloadAttributesHelper struct { + slot uint64 + parentHash string + withdrawalsRoot phase0.Root + payloadAttributes beaconclient.PayloadAttributes } // RelayAPI represents a single Relay instance @@ -157,22 +154,14 @@ type RelayAPI struct { getPayloadCallsInFlight sync.WaitGroup // Feature flags - ffForceGetHeader204 bool - ffDisableLowPrioBuilders bool - ffDisablePayloadDBStorage bool // disable storing the execution payloads in the database - ffDisableSSEPayloadAttributes bool // instead of SSE, fall back to previous polling withdrawals+prevRandao from our custom Prysm fork - ffAllowMemcacheSavingFail bool // don't fail when saving payloads to memcache doesn't succeed - ffLogInvalidSignaturePayload bool // log payload if getPayload signature validation fails - - latestParentBlockHash uberatomic.String // used to cache the latest parent block hash, to avoid repetitive similar SSE events - - expectedPrevRandao randaoHelper - expectedPrevRandaoLock sync.RWMutex - expectedPrevRandaoUpdating uint64 - - expectedWithdrawalsRoot withdrawalsHelper - expectedWithdrawalsLock sync.RWMutex - expectedWithdrawalsUpdating uint64 + ffForceGetHeader204 bool + ffDisableLowPrioBuilders bool + ffDisablePayloadDBStorage bool // disable storing the execution payloads in the database + ffAllowMemcacheSavingFail bool // don't fail when saving payloads to memcache doesn't succeed + ffLogInvalidSignaturePayload bool // log payload if getPayload signature validation fails + + payloadAttributes map[string]payloadAttributesHelper // key:parentBlockHash + payloadAttributesLock sync.RWMutex } // NewRelayAPI creates a new service. if builders is nil, allow any builder @@ -222,15 +211,17 @@ func NewRelayAPI(opts RelayAPIOpts) (api *RelayAPI, err error) { } api = &RelayAPI{ - opts: opts, - log: opts.Log, - blsSk: opts.SecretKey, - publicKey: &publicKey, - datastore: opts.Datastore, - beaconClient: opts.BeaconClient, - redis: opts.Redis, - memcached: opts.Memcached, - db: opts.DB, + opts: opts, + log: opts.Log, + blsSk: opts.SecretKey, + publicKey: &publicKey, + datastore: opts.Datastore, + beaconClient: opts.BeaconClient, + redis: opts.Redis, + memcached: opts.Memcached, + db: opts.DB, + + payloadAttributes: make(map[string]payloadAttributesHelper), proposerDutiesResponse: []boostTypes.BuilderGetValidatorsResponseEntry{}, blockSimRateLimiter: NewBlockSimulationRateLimiter(opts.BlockSimURL), @@ -253,11 +244,6 @@ func NewRelayAPI(opts RelayAPIOpts) (api *RelayAPI, err error) { api.ffDisablePayloadDBStorage = true } - if os.Getenv("DISABLE_SSE_PAYLOAD_ATTRIBUTES") == "1" { - api.log.Warn("env: DISABLE_SSE_PAYLOAD_ATTRIBUTES - using previous polling logic for withdrawals and randao (requires custom Prysm fork)") - api.ffDisableSSEPayloadAttributes = true - } - if os.Getenv("MEMCACHE_ALLOW_SAVING_FAIL") == "1" { api.log.Warn("env: MEMCACHE_ALLOW_SAVING_FAIL - continue block submission request even if saving to memcache fails") api.ffAllowMemcacheSavingFail = true @@ -419,7 +405,7 @@ func (api *RelayAPI) StartServer() (err error) { // Start regular payload attributes updates only if builder-api is enabled // and if using see subscriptions instead of querying for payload attributes - if api.opts.BlockBuilderAPI && !api.ffDisableSSEPayloadAttributes { + if api.opts.BlockBuilderAPI { go func() { c := make(chan beaconclient.PayloadAttributesEvent) api.beaconClient.SubscribeToPayloadAttributesEvents(c) @@ -495,52 +481,60 @@ func (api *RelayAPI) startValidatorRegistrationDBProcessor() { func (api *RelayAPI) processPayloadAttributes(payloadAttributes beaconclient.PayloadAttributesEvent) { apiHeadSlot := api.headSlot.Load() - proposalSlot := payloadAttributes.Data.ProposalSlot + payloadAttrSlot := payloadAttributes.Data.ProposalSlot // require proposal slot in the future - if proposalSlot <= apiHeadSlot { + if payloadAttrSlot <= apiHeadSlot { return } log := api.log.WithFields(logrus.Fields{ - "headSlot": apiHeadSlot, - "proposalSlot": proposalSlot, + "headSlot": apiHeadSlot, + "payloadAttrSlot": payloadAttrSlot, + "payloadAttrParent": payloadAttributes.Data.ParentBlockHash, }) - // discard repetitive payload attributes (we receive them once from each beacon node) - latestParentBlockHash := api.latestParentBlockHash.Load() - if latestParentBlockHash == payloadAttributes.Data.ParentBlockHash { + // discard payload attributes if already known + api.payloadAttributesLock.RLock() + _, ok := api.payloadAttributes[payloadAttributes.Data.ParentBlockHash] + api.payloadAttributesLock.RUnlock() + + if ok { return } - api.latestParentBlockHash.Store(payloadAttributes.Data.ParentBlockHash) - log = log.WithField("parentBlockHash", payloadAttributes.Data.ParentBlockHash) - log.Info("updating payload attributes") - api.expectedPrevRandaoLock.Lock() - prevRandao := payloadAttributes.Data.PayloadAttributes.PrevRandao - api.expectedPrevRandao = randaoHelper{ - slot: proposalSlot, - prevRandao: prevRandao, + var withdrawalsRoot phase0.Root + var err error + if api.isCapella(payloadAttrSlot) { + withdrawalsRoot, err = ComputeWithdrawalsRoot(payloadAttributes.Data.PayloadAttributes.Withdrawals) + log = log.WithField("withdrawalsRoot", withdrawalsRoot.String()) + if err != nil { + log.WithError(err).Error("error computing withdrawals root") + return + } } - api.expectedPrevRandaoLock.Unlock() - log.Infof("updated expected prev_randao to %s", prevRandao) - // Update withdrawals (in Capella only) - if api.isBellatrix(proposalSlot) { - return - } - log.Info("updating expected withdrawals") - withdrawalsRoot, err := ComputeWithdrawalsRoot(payloadAttributes.Data.PayloadAttributes.Withdrawals) - if err != nil { - log.WithError(err).Error("error computing withdrawals root") - return + api.payloadAttributesLock.Lock() + defer api.payloadAttributesLock.Unlock() + + // Step 1: clean up old ones + for parentBlockHash, attr := range api.payloadAttributes { + if attr.slot < apiHeadSlot { + delete(api.payloadAttributes, parentBlockHash) + } } - api.expectedWithdrawalsLock.Lock() - api.expectedWithdrawalsRoot = withdrawalsHelper{ - slot: proposalSlot, - root: withdrawalsRoot, + + // Step 2: save new one + api.payloadAttributes[payloadAttributes.Data.ParentBlockHash] = payloadAttributesHelper{ + slot: payloadAttrSlot, + parentHash: payloadAttributes.Data.ParentBlockHash, + withdrawalsRoot: withdrawalsRoot, + payloadAttributes: payloadAttributes.Data.PayloadAttributes, } - api.expectedWithdrawalsLock.Unlock() - log.Infof("updated expected withdrawals root to %s", withdrawalsRoot) + + log.WithFields(logrus.Fields{ + "randao": payloadAttributes.Data.PayloadAttributes.PrevRandao, + "timestamp": payloadAttributes.Data.PayloadAttributes.Timestamp, + }).Info("updated payload attributes") } func (api *RelayAPI) processNewSlot(headSlot uint64) { @@ -561,15 +555,6 @@ func (api *RelayAPI) processNewSlot(headSlot uint64) { // only for builder-api if api.opts.BlockBuilderAPI { - // if not subscribed to payload attributes via sse, query beacon node endpoints - if api.ffDisableSSEPayloadAttributes { - // query the expected prev_randao field - go api.updatedExpectedRandao(headSlot) - - // query expected withdrawals root - go api.updatedExpectedWithdrawals(headSlot) - } - // update proposer duties in the background go api.updateProposerDuties(headSlot) } @@ -1134,105 +1119,10 @@ func (api *RelayAPI) handleGetPayload(w http.ResponseWriter, req *http.Request) } // -------------------- -// BLOCK BUILDER APIS +// +// BLOCK BUILDER APIS +// // -------------------- - -// updatedExpectedRandao updates the prev_randao field we expect from builder block submissions -func (api *RelayAPI) updatedExpectedRandao(slot uint64) { - log := api.log.WithField("slot", slot) - log.Infof("updating randao...") - api.expectedPrevRandaoLock.Lock() - latestKnownSlot := api.expectedPrevRandao.slot - if slot < latestKnownSlot || slot <= api.expectedPrevRandaoUpdating { // do nothing slot is already known or currently being updated - log.Debugf("- abort updating randao, latest: %d, updating: %d", latestKnownSlot, api.expectedPrevRandaoUpdating) - api.expectedPrevRandaoLock.Unlock() - return - } - api.expectedPrevRandaoUpdating = slot - api.expectedPrevRandaoLock.Unlock() - - // get randao from BN - log.Debugf("- querying BN for randao") - randao, err := api.beaconClient.GetRandao(slot) - if err != nil { - log.WithError(err).Error("failed to get randao from beacon node") - api.expectedPrevRandaoLock.Lock() - api.expectedPrevRandaoUpdating = 0 - api.expectedPrevRandaoLock.Unlock() - return - } - - // after request, check if still the latest, then update - api.expectedPrevRandaoLock.Lock() - defer api.expectedPrevRandaoLock.Unlock() - targetSlot := slot + 1 - log.Debugf("- after BN randao: targetSlot: %d latest: %d", targetSlot, api.expectedPrevRandao.slot) - - // update if still the latest - if targetSlot >= api.expectedPrevRandao.slot { - api.expectedPrevRandao = randaoHelper{ - slot: targetSlot, // the retrieved prev_randao is for the next slot - prevRandao: randao.Data.Randao, - } - log.Infof("updated expected prev_randao to %s for slot %d", randao.Data.Randao, targetSlot) - } -} - -// updatedExpectedWithdrawals updates the withdrawals field we expect from builder block submissions -func (api *RelayAPI) updatedExpectedWithdrawals(slot uint64) { - if api.isBellatrix(slot) { - return - } - - log := api.log.WithField("slot", slot) - log.Info("updating withdrawals root...") - api.expectedWithdrawalsLock.Lock() - latestKnownSlot := api.expectedWithdrawalsRoot.slot - if slot < latestKnownSlot || slot <= api.expectedWithdrawalsUpdating { // do nothing slot is already known or currently being updated - log.Debugf("- abort updating withdrawals root, latest: %d, updating: %d", latestKnownSlot, api.expectedWithdrawalsUpdating) - api.expectedWithdrawalsLock.Unlock() - return - } - api.expectedWithdrawalsUpdating = slot - api.expectedWithdrawalsLock.Unlock() - - // get withdrawals from BN - log.Debugf("- querying BN for withdrawals for slot %d", slot) - withdrawals, err := api.beaconClient.GetWithdrawals(slot) - if err != nil { - if errors.Is(err, beaconclient.ErrWithdrawalsBeforeCapella) { - log.WithError(err).Debug("attempted to fetch withdrawals before capella") - } else { - log.WithError(err).Error("failed to get withdrawals from beacon node") - } - api.expectedWithdrawalsLock.Lock() - api.expectedWithdrawalsUpdating = 0 - api.expectedWithdrawalsLock.Unlock() - return - } - - // after request, check if still the latest, then update - api.expectedWithdrawalsLock.Lock() - defer api.expectedWithdrawalsLock.Unlock() - targetSlot := slot + 1 - log.Debugf("- after BN withdrawals: targetSlot: %d latest: %d", targetSlot, api.expectedWithdrawalsRoot.slot) - - // update if still the latest - if targetSlot >= api.expectedWithdrawalsRoot.slot { - withdrawalsRoot, err := ComputeWithdrawalsRoot(withdrawals.Data.Withdrawals) - if err != nil { - log.WithError(err).Warn("failed to compute withdrawals root") - api.expectedWithdrawalsUpdating = 0 - return - } - api.expectedWithdrawalsRoot = withdrawalsHelper{ - slot: targetSlot, // the retrieved withdrawals is for the next slot - root: withdrawalsRoot, - } - log.Infof("updated expected withdrawals root to %s for slot %d", withdrawalsRoot, targetSlot) - } -} - func (api *RelayAPI) handleBuilderGetValidators(w http.ResponseWriter, req *http.Request) { api.proposerDutiesLock.RLock() defer api.proposerDutiesLock.RUnlock() @@ -1379,39 +1269,32 @@ func (api *RelayAPI) handleSubmitNewBlock(w http.ResponseWriter, req *http.Reque return } - // get the latest randao and check its slot - api.expectedPrevRandaoLock.RLock() - expectedRandao := api.expectedPrevRandao - api.expectedPrevRandaoLock.RUnlock() - if expectedRandao.slot != payload.Slot() { - log.Warn("prev_randao is not known yet") - api.RespondError(w, http.StatusInternalServerError, "prev_randao is not known yet") + api.payloadAttributesLock.RLock() + attrs, ok := api.payloadAttributes[payload.ParentHash()] + api.payloadAttributesLock.RUnlock() + if !ok || payload.Slot() != attrs.slot { + log.Warn("payload attributes not (yet) known") + api.RespondError(w, http.StatusInternalServerError, "payload attributes not (yet) known") return - } else if expectedRandao.prevRandao != payload.Random() { - msg := fmt.Sprintf("incorrect prev_randao - got: %s, expected: %s", payload.Random(), expectedRandao.prevRandao) + } + + if payload.Random() != attrs.payloadAttributes.PrevRandao { + msg := fmt.Sprintf("incorrect prev_randao - got: %s, expected: %s", payload.Random(), attrs.payloadAttributes.PrevRandao) log.Info(msg) api.RespondError(w, http.StatusBadRequest, msg) return } - withdrawals := payload.Withdrawals() - if withdrawals != nil { - // get latest withdrawals and verify the roots match - api.expectedWithdrawalsLock.RLock() - expectedWithdrawalsRoot := api.expectedWithdrawalsRoot - api.expectedWithdrawalsLock.RUnlock() + if api.isCapella(payload.Slot()) { // Capella requires correct withdrawals withdrawalsRoot, err := ComputeWithdrawalsRoot(payload.Withdrawals()) if err != nil { log.WithError(err).Warn("could not compute withdrawals root from payload") api.RespondError(w, http.StatusBadRequest, "could not compute withdrawals root") return } - if expectedWithdrawalsRoot.slot != payload.Slot() { - log.Warn("withdrawals are not known yet") - api.RespondError(w, http.StatusInternalServerError, "withdrawals are not known yet") - return - } else if expectedWithdrawalsRoot.root != withdrawalsRoot { - msg := fmt.Sprintf("incorrect withdrawals root - got: %s, expected: %s", withdrawalsRoot.String(), expectedWithdrawalsRoot.root.String()) + + if withdrawalsRoot != attrs.withdrawalsRoot { + msg := fmt.Sprintf("incorrect withdrawals root - got: %s, expected: %s", withdrawalsRoot.String(), attrs.withdrawalsRoot.String()) log.Info(msg) api.RespondError(w, http.StatusBadRequest, msg) return @@ -1421,7 +1304,7 @@ func (api *RelayAPI) handleSubmitNewBlock(w http.ResponseWriter, req *http.Reque // Verify the signature builderPubkey := payload.BuilderPubkey() signature := payload.Signature() - ok, err := boostTypes.VerifySignature(payload.Message(), api.opts.EthNetDetails.DomainBuilder, builderPubkey[:], signature[:]) + ok, err = boostTypes.VerifySignature(payload.Message(), api.opts.EthNetDetails.DomainBuilder, builderPubkey[:], signature[:]) if !ok || err != nil { log.WithError(err).Warn("could not verify builder signature") api.RespondError(w, http.StatusBadRequest, "invalid signature") diff --git a/services/api/utils.go b/services/api/utils.go index fcc89d0b..fe1e765f 100644 --- a/services/api/utils.go +++ b/services/api/utils.go @@ -15,6 +15,7 @@ var ( ErrParentHashMismatch = errors.New("parentHash mismatch") ErrNoPayloads = errors.New("no payloads") + ErrNoWithdrawals = errors.New("no withdrawals") ErrPayloadMismatchBellatrix = errors.New("bellatrix beacon-block but no bellatrix payload") ErrPayloadMismatchCapella = errors.New("capella beacon-block but no capella payload") ErrHeaderHTRMismatch = errors.New("beacon-block and payload header mismatch") @@ -38,6 +39,9 @@ func checkBLSPublicKeyHex(pkHex string) error { } func ComputeWithdrawalsRoot(w []*capella.Withdrawal) (phase0.Root, error) { + if w == nil { + return phase0.Root{}, ErrNoWithdrawals + } withdrawals := utilcapella.ExecutionPayloadWithdrawals{Withdrawals: w} return withdrawals.HashTreeRoot() }