go-algorand v3.9.2-beta

buildnumber.dat

@@ -1 +1 @@
-1
+2

catchup/catchpointService.go

@@ -488,10 +488,14 @@ func (cs *CatchpointCatchupService) processStageBlocksDownload() (err error) {
 		return cs.abort(fmt.Errorf("processStageBlocksDownload failed, unable to ensure first block : %v", err))
 	}
 
-	// pick the lookback with the greater of either (MaxTxnLife+DeeperBlockHeaderHistory)
-	// or MaxBalLookback
+	// pick the lookback with the greater of
+	// either (MaxTxnLife+DeeperBlockHeaderHistory+CatchpointLookback) or MaxBalLookback
+	// Explanation:
+	// 1. catchpoint snapshots accounts at round X-CatchpointLookback
+	// 2. replay starts from X-CatchpointLookback+1
+	// 3. transaction evaluation at Y requires block up to MaxTxnLife+DeeperBlockHeaderHistory back from Y
 	proto := config.Consensus[topBlock.CurrentProtocol]
-	lookback := proto.MaxTxnLife + proto.DeeperBlockHeaderHistory
+	lookback := proto.MaxTxnLife + proto.DeeperBlockHeaderHistory + proto.CatchpointLookback
 	if lookback < proto.MaxBalLookback {
 		lookback = proto.MaxBalLookback
 	}

config/consensus.go

@@ -392,6 +392,10 @@ type ConsensusParams struct {
 	// release resources allocated for creating state proofs.
 	StateProofMaxRecoveryIntervals uint64
 
+	// StateProofExcludeTotalWeightWithRewards specifies whether to subtract rewards from excluded online accounts along with
+	// their account balances.
+	StateProofExcludeTotalWeightWithRewards bool
+
 	// EnableAssetCloseAmount adds an extra field to the ApplyData. The field contains the amount of the remaining
 	// asset that were sent to the close-to address.
 	EnableAssetCloseAmount bool
@@ -1192,14 +1196,22 @@ func initConsensusProtocols() {
 
 	Consensus[protocol.ConsensusV34] = v34
 
-	// v33 can be upgraded to v34, with an update delay of 12h:
+	v35 := v34
+	v35.StateProofExcludeTotalWeightWithRewards = true
+
+	v35.ApprovedUpgrades = map[protocol.ConsensusVersion]uint64{}
+
+	Consensus[protocol.ConsensusV35] = v35
+
+	// v33 and v34 can be upgraded to v35, with an update delay of 12h:
 	// 10046 = (12 * 60 * 60 / 4.3)
 	// for the sake of future manual calculations, we'll round that down a bit :
-	v33.ApprovedUpgrades[protocol.ConsensusV34] = 10000
+	v33.ApprovedUpgrades[protocol.ConsensusV35] = 10000
+	v34.ApprovedUpgrades[protocol.ConsensusV35] = 10000
 
 	// ConsensusFuture is used to test features that are implemented
 	// but not yet released in a production protocol version.
-	vFuture := v34
+	vFuture := v35
 	vFuture.ApprovedUpgrades = map[protocol.ConsensusVersion]uint64{}
 
 	vFuture.LogicSigVersion = 8 // When moving this to a release, put a new higher LogicSigVersion here

ledger/accountdb.go

@@ -2136,7 +2136,8 @@ func performTxTailTableMigration(ctx context.Context, tx *sql.Tx, blockDb db.Acc
 		}
 
 		maxTxnLife := basics.Round(config.Consensus[latestHdr.CurrentProtocol].MaxTxnLife)
-		firstRound := (latestBlockRound + 1).SubSaturate(maxTxnLife)
+		deeperBlockHistory := basics.Round(config.Consensus[latestHdr.CurrentProtocol].DeeperBlockHeaderHistory)
+		firstRound := (latestBlockRound + 1).SubSaturate(maxTxnLife + deeperBlockHistory)
 		// we don't need to have the txtail for round 0.
 		if firstRound == basics.Round(0) {
 			firstRound++

ledger/acctonline.go

@@ -748,7 +748,7 @@ func (ao *onlineAccounts) lookupOnlineAccountData(rnd basics.Round, addr basics.
 // not participate in round == voteRnd.
 // See the normalization description in AccountData.NormalizedOnlineBalance().
 // The return value of totalOnlineStake represents the total stake that is online for voteRnd: it is an approximation since voteRnd did not yet occur.
-func (ao *onlineAccounts) TopOnlineAccounts(rnd basics.Round, voteRnd basics.Round, n uint64) (topOnlineAccounts []*ledgercore.OnlineAccount, totalOnlineStake basics.MicroAlgos, err error) {
+func (ao *onlineAccounts) TopOnlineAccounts(rnd basics.Round, voteRnd basics.Round, n uint64, params *config.ConsensusParams, rewardsLevel uint64) (topOnlineAccounts []*ledgercore.OnlineAccount, totalOnlineStake basics.MicroAlgos, err error) {
 	genesisProto := ao.ledger.GenesisProto()
 	ao.accountsMu.RLock()
 	for {
@@ -904,6 +904,13 @@ func (ao *onlineAccounts) TopOnlineAccounts(rnd basics.Round, voteRnd basics.Rou
 			if ot.Overflowed {
 				return nil, basics.MicroAlgos{}, fmt.Errorf("TopOnlineAccounts: overflow in stakeOfflineInVoteRound")
 			}
+			if params.StateProofExcludeTotalWeightWithRewards {
+				rewards := basics.PendingRewards(&ot, *params, oa.MicroAlgos, oa.RewardsBase, rewardsLevel)
+				totalOnlineStake = ot.SubA(totalOnlineStake, rewards)
+				if ot.Overflowed {
+					return nil, basics.MicroAlgos{}, fmt.Errorf("TopOnlineAccounts: overflow in stakeOfflineInVoteRound rewards")
+				}
+			}
 		}
 
 		return topOnlineAccounts, totalOnlineStake, nil

ledger/acctonline_test.go

@@ -1392,7 +1392,8 @@ func TestAcctOnlineTopInBatches(t *testing.T) {
 	_, oa := newAcctUpdates(t, ml, conf)
 	defer oa.close()
 
-	top, _, err := oa.TopOnlineAccounts(0, 0, 2048)
+	proto := config.Consensus[protocol.ConsensusCurrentVersion]
+	top, _, err := oa.TopOnlineAccounts(0, 0, 2048, &proto, 0)
 	a.NoError(err)
 	compareTopAccounts(a, top, allAccts)
 }
@@ -1437,7 +1438,8 @@ func TestAcctOnlineTopBetweenCommitAndPostCommit(t *testing.T) {
 	defer oa.close()
 	ml.trackers.trackers = append([]ledgerTracker{stallingTracker}, ml.trackers.trackers...)
 
-	top, _, err := oa.TopOnlineAccounts(0, 0, 5)
+	proto := config.Consensus[protocol.ConsensusCurrentVersion]
+	top, _, err := oa.TopOnlineAccounts(0, 0, 5, &proto, 0)
 	a.NoError(err)
 	compareTopAccounts(a, top, allAccts)
 
@@ -1475,7 +1477,8 @@ func TestAcctOnlineTopBetweenCommitAndPostCommit(t *testing.T) {
 			time.Sleep(2 * time.Second)
 			stallingTracker.postCommitReleaseLock <- struct{}{}
 		}()
-		top, _, err = oa.TopOnlineAccounts(2, 2, 5)
+
+		top, _, err = oa.TopOnlineAccounts(2, 2, 5, &proto, 0)
 		a.NoError(err)
 
 		accountToBeUpdated := allAccts[numAccts-1]
@@ -1528,7 +1531,8 @@ func TestAcctOnlineTopDBBehindMemRound(t *testing.T) {
 	defer oa.close()
 	ml.trackers.trackers = append([]ledgerTracker{stallingTracker}, ml.trackers.trackers...)
 
-	top, _, err := oa.TopOnlineAccounts(0, 0, 5)
+	proto := config.Consensus[protocol.ConsensusCurrentVersion]
+	top, _, err := oa.TopOnlineAccounts(0, 0, 5, &proto, 0)
 	a.NoError(err)
 	compareTopAccounts(a, top, allAccts)
 
@@ -1571,7 +1575,8 @@ func TestAcctOnlineTopDBBehindMemRound(t *testing.T) {
 			})
 			stallingTracker.postCommitReleaseLock <- struct{}{}
 		}()
-		_, _, err = oa.TopOnlineAccounts(2, 2, 5)
+
+		_, _, err = oa.TopOnlineAccounts(2, 2, 5, &proto, 0)
 		a.Error(err)
 		a.Contains(err.Error(), "is behind in-memory round")
 
@@ -1680,7 +1685,8 @@ func (m *MicroAlgoOperations) Add(x, y basics.MicroAlgos) basics.MicroAlgos {
 }
 
 func compareOnlineTotals(a *require.Assertions, oa *onlineAccounts, rnd, voteRnd basics.Round, n uint64, expectedForRnd, expectedForVoteRnd basics.MicroAlgos) []*ledgercore.OnlineAccount {
-	top, onlineTotalVoteRnd, err := oa.TopOnlineAccounts(rnd, voteRnd, n)
+	proto := config.Consensus[protocol.ConsensusCurrentVersion]
+	top, onlineTotalVoteRnd, err := oa.TopOnlineAccounts(rnd, voteRnd, n, &proto, 0)
 	a.NoError(err)
 	a.Equal(expectedForVoteRnd, onlineTotalVoteRnd)
 	onlineTotalsRnd, err := oa.onlineTotals(rnd)

ledger/applications_test.go

@@ -36,15 +36,15 @@ import (
 // commitRound schedules a commit for known offset and dbRound
 // and waits for completion
 func commitRound(offset uint64, dbRound basics.Round, l *Ledger) {
-	commitRoundLookback(dbRound+basics.Round(offset), l)
+	commitRoundLookback(l.Latest().SubSaturate(dbRound+basics.Round(offset)), l)
 }
 
 func commitRoundLookback(lookback basics.Round, l *Ledger) {
 	l.trackers.mu.Lock()
 	l.trackers.lastFlushTime = time.Time{}
 	l.trackers.mu.Unlock()
 
-	l.trackers.scheduleCommit(l.Latest(), l.Latest()-lookback)
+	l.trackers.scheduleCommit(l.Latest(), lookback)
 	// wait for the operation to complete. Once it does complete, the tr.lastFlushTime is going to be updated, so we can
 	// use that as an indicator.
 	for {

ledger/internal/apptxn_test.go

@@ -3295,3 +3295,33 @@ done:
 		require.Equal(t, "Y", vb.Block().Payset[3].EvalDelta.LocalDeltas[1]["X"].Bytes)
 	})
 }
+
+// TestReloadWithTxns confirms that the ledger can be reloaded from "disk" when
+// doing so requires replaying some interesting AVM txns.
+func TestReloadWithTxns(t *testing.T) {
+	partitiontest.PartitionTest(t)
+
+	genBalances, addrs, _ := ledgertesting.NewTestGenesis()
+	testConsensusRange(t, 34, 0, func(t *testing.T, ver int) {
+		fmt.Printf("testConsensus %d\n", ver)
+		dl := NewDoubleLedger(t, genBalances, consensusByNumber[ver])
+		defer dl.Close()
+
+		dl.fullBlock() // So that the `block` opcode has a block to inspect
+
+		lookHdr := txntest.Txn{
+			Type:   "appl",
+			Sender: addrs[0],
+			ApprovalProgram: `
+			  txn FirstValid
+			  int 1
+			  -
+			  block BlkTimestamp
+`,
+		}
+
+		dl.fullBlock(&lookHdr)
+
+		dl.reloadLedgers()
+	})
+}

ledger/internal/double_test.go

@@ -129,6 +129,11 @@ func (dl *DoubleLedger) endBlock() *ledgercore.ValidatedBlock {
 	return vb
 }
 
+func (dl *DoubleLedger) reloadLedgers() {
+	require.NoError(dl.t, dl.generator.ReloadLedger())
+	require.NoError(dl.t, dl.validator.ReloadLedger())
+}
+
 func checkBlock(t *testing.T, checkLedger *ledger.Ledger, vb *ledgercore.ValidatedBlock) {
 	bl := vb.Block()
 	msg := bl.MarshalMsg(nil)

ledger/internal/eval_blackbox_test.go

@@ -900,6 +900,7 @@ var consensusByNumber = []protocol.ConsensusVersion{
 	protocol.ConsensusV32, // unlimited assets and apps
 	protocol.ConsensusV33, // 320 rounds
 	protocol.ConsensusV34, // AVM v7, stateproofs
+	protocol.ConsensusV35, // stateproofs stake fix
 	protocol.ConsensusFuture,
 }
 

ledger/ledger.go

@@ -167,6 +167,12 @@ func OpenLedger(
 	return l, nil
 }
 
+// ReloadLedger is exported for the benefit of tests in the internal
+// package. Revisit this when we rename / restructure that thing
+func (l *Ledger) ReloadLedger() error {
+	return l.reloadLedger()
+}
+
 func (l *Ledger) reloadLedger() error {
 	// similar to the Close function, we want to start by closing the blockQ first. The
 	// blockQ is having a sync goroutine which indirectly calls other trackers. We want to eliminate that go-routine first,