Algod: Add EvalTracer tests for StateDeltas

data/transactions/logic/mocktracer/tracer.go

@@ -17,12 +17,16 @@
 package mocktracer
 
 import (
+	"testing"
+
 	"github.com/algorand/go-algorand/data/basics"
 	"github.com/algorand/go-algorand/data/bookkeeping"
 	"github.com/algorand/go-algorand/data/transactions"
 	"github.com/algorand/go-algorand/data/transactions/logic"
 	"github.com/algorand/go-algorand/ledger/ledgercore"
 	"github.com/algorand/go-algorand/protocol"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 // EventType represents a type of logic.EvalTracer event
@@ -64,6 +68,9 @@ type Event struct {
 	// only for AfterTxn
 	TxnApplyData transactions.ApplyData
 
+	// only for AfterTxnGroup and AfterTxn
+	Deltas *ledgercore.StateDelta
+
 	// only for BeforeTxnGroup and AfterTxnGroup
 	GroupSize int
 
@@ -85,8 +92,8 @@ func BeforeTxnGroup(groupSize int) Event {
 }
 
 // AfterTxnGroup creates a new Event with the type AfterTxnGroupEvent
-func AfterTxnGroup(groupSize int, hasError bool) Event {
-	return Event{Type: AfterTxnGroupEvent, GroupSize: groupSize, HasError: hasError}
+func AfterTxnGroup(groupSize int, deltas *ledgercore.StateDelta, hasError bool) Event {
+	return Event{Type: AfterTxnGroupEvent, GroupSize: groupSize, Deltas: copyDeltas(deltas), HasError: hasError}
 }
 
 // BeforeProgram creates a new Event with the type BeforeProgramEvent
@@ -163,7 +170,7 @@ func (d *Tracer) BeforeTxnGroup(ep *logic.EvalParams) {
 
 // AfterTxnGroup mocks the logic.EvalTracer.AfterTxnGroup method
 func (d *Tracer) AfterTxnGroup(ep *logic.EvalParams, deltas *ledgercore.StateDelta, evalError error) {
-	d.Events = append(d.Events, AfterTxnGroup(len(ep.TxnGroup), evalError != nil))
+	d.Events = append(d.Events, AfterTxnGroup(len(ep.TxnGroup), deltas, evalError != nil))
 }
 
 // BeforeTxn mocks the logic.EvalTracer.BeforeTxn method
@@ -200,3 +207,47 @@ func (d *Tracer) AfterOpcode(cx *logic.EvalContext, evalError error) {
 func (d *Tracer) AfterBlock(hdr *bookkeeping.BlockHeader) {
 	d.Events = append(d.Events, AfterBlock(hdr.Round))
 }
+
+// copyDeltas makes a deep copy of the given ledgercore.StateDelta pointer, if it's not nil.
+// This is inefficient, but it should only be used for testing.
+func copyDeltas(deltas *ledgercore.StateDelta) *ledgercore.StateDelta {
+	if deltas == nil {
+		return nil
+	}
+	encoded := protocol.EncodeReflect(deltas)
+	var clone ledgercore.StateDelta
+	err := protocol.DecodeReflect(encoded, &clone)
+	if err != nil {
+		panic(err)
+	}
+	return &clone
+}
+
+// AssertEventsEqual asserts that two slices of Events are equal, taking into account complex
+// equality of StateDeltas. The arguments will be modified in-place to normalize any StateDeltas.
+func AssertEventsEqual(t *testing.T, expected, actual []Event) {
+	t.Helper()
+
+	// Dehydrate deltas for better comparison
+	for i := range expected {
+		if expected[i].Deltas != nil {
+			expected[i].Deltas.Dehydrate()
+		}
+	}
+	for i := range actual {
+		if actual[i].Deltas != nil {
+			actual[i].Deltas.Dehydrate()
+		}
+	}
+
+	// These extra checks are not necessary for correctness, but they provide more targeted information on failure
+	if assert.Equal(t, len(expected), len(actual)) {
+		for i := range expected {
+			jsonExpectedDelta := protocol.EncodeJSONStrict(expected[i].Deltas)
+			jsonActualDelta := protocol.EncodeJSONStrict(actual[i].Deltas)
+			assert.Equal(t, expected[i].Deltas, actual[i].Deltas, "StateDelta disagreement: i=%d, event type: (%v,%v)\n\nexpected: %s\n\nactual: %s", i, expected[i].Type, actual[i].Type, jsonExpectedDelta, jsonActualDelta)
+		}
+	}
+
+	require.Equal(t, expected, actual)
+}

ledger/eval/cow_test.go

@@ -192,6 +192,51 @@ func TestCowBalance(t *testing.T) {
 	checkCowByUpdate(t, c0, updates2)
 }
 
+// TestCowDeltasAfterCommit tests that deltas are still valid after committing to parent.
+func TestCowDeltasAfterCommit(t *testing.T) {
+	partitiontest.PartitionTest(t)
+	t.Parallel()
+
+	accts0 := ledgertesting.RandomAccounts(20, true)
+	ml := mockLedger{balanceMap: accts0}
+
+	c0 := makeRoundCowState(
+		&ml, bookkeeping.BlockHeader{}, config.Consensus[protocol.ConsensusCurrentVersion],
+		0, ledgercore.AccountTotals{}, 0)
+	checkCow(t, c0, accts0)
+
+	c1 := c0.child(0)
+
+	acctUpdates, _, _ := ledgertesting.RandomDeltas(10, accts0, 0)
+	applyUpdates(c1, acctUpdates)
+	acctUpdates.Dehydrate() // Prep for comparison
+
+	c1.kvPut("key", []byte("value"))
+	expectedKvMods := map[string]ledgercore.KvValueDelta{
+		"key": {
+			Data: []byte("value"),
+		},
+	}
+
+	actualDeltas := c1.deltas()
+	actualDeltas.Dehydrate() // Prep for comparison
+	require.Equal(t, acctUpdates, actualDeltas.Accts)
+	require.Equal(t, expectedKvMods, actualDeltas.KvMods)
+
+	// Parent should now have deltas
+	c1.commitToParent()
+	actualDeltas = c0.deltas()
+	actualDeltas.Dehydrate() // Prep for comparison
+	require.Equal(t, acctUpdates, actualDeltas.Accts)
+	require.Equal(t, expectedKvMods, actualDeltas.KvMods)
+
+	// Deltas remain valid in child after commit
+	actualDeltas = c0.deltas()
+	actualDeltas.Dehydrate() // Prep for comparison
+	require.Equal(t, acctUpdates, actualDeltas.Accts)
+	require.Equal(t, expectedKvMods, actualDeltas.KvMods)
+}
+
 func BenchmarkCowChild(b *testing.B) {
 	b.ReportAllocs()
 	cow := makeRoundCowState(nil, bookkeeping.BlockHeader{}, config.ConsensusParams{}, 10000, ledgercore.AccountTotals{}, 16)

ledger/eval/eval_test.go

@@ -275,13 +275,14 @@ func TestTransactionGroupWithTracer(t *testing.T) {
 			t.Parallel()
 			genesisInitState, addrs, keys := ledgertesting.Genesis(10)
 
+			basicAppID := basics.AppIndex(1001)
 			innerAppID := basics.AppIndex(1003)
 			innerAppAddress := innerAppID.Address()
 			balances := genesisInitState.Accounts
 			balances[innerAppAddress] = basics_testing.MakeAccountData(basics.Offline, basics.MicroAlgos{Raw: 1_000_000})
 
 			genesisBalances := bookkeeping.GenesisBalances{
-				Balances:    genesisInitState.Accounts,
+				Balances:    balances,
 				FeeSink:     testSinkAddr,
 				RewardsPool: testPoolAddr,
 				Timestamp:   0,
@@ -343,7 +344,7 @@ int 1`,
 			// an app call with inner txn
 			innerAppCallTxn := txntest.Txn{
 				Type:   protocol.ApplicationCallTx,
-				Sender: addrs[0],
+				Sender: addrs[4],
 				ClearStateProgram: `#pragma version 6
 int 1`,
 
@@ -352,19 +353,50 @@ int 1`,
 				Fee:         minFee,
 				GenesisHash: genHash,
 			}
+
+			expectedFeeSinkDataForScenario := ledgercore.ToAccountData(balances[testSinkAddr])
+			expectedFeeSinkDataForScenario.MicroAlgos.Raw += basicAppCallTxn.Txn().Fee.Raw
+			if testCase.firstTxnBehavior == "approve" {
+				expectedFeeSinkDataForScenario.MicroAlgos.Raw += payTxn.Txn().Fee.Raw
+			}
+
 			scenario := testCase.innerAppCallScenario(mocktracer.TestScenarioInfo{
-				CallingTxn:   innerAppCallTxn.Txn(),
-				MinFee:       minFee,
-				CreatedAppID: innerAppID,
+				CallingTxn:     innerAppCallTxn.Txn(),
+				SenderData:     ledgercore.ToAccountData(balances[addrs[4]]),
+				AppAccountData: ledgercore.ToAccountData(balances[innerAppAddress]),
+				FeeSinkData:    expectedFeeSinkDataForScenario,
+				FeeSinkAddr:    testSinkAddr,
+				MinFee:         minFee,
+				CreatedAppID:   innerAppID,
+				BlockHeader:    eval.block.BlockHeader,
+				PrevTimestamp:  blkHeader.TimeStamp,
 			})
 			innerAppCallTxn.ApprovalProgram = scenario.Program
 
 			txntest.Group(&basicAppCallTxn, &payTxn, &innerAppCallTxn)
 
+			// Update the expected state delta to reflect the inner app call txid
+			scenarioTxidValue, ok := scenario.ExpectedStateDelta.Txids[transactions.Txid{}]
+			if ok {
+				delete(scenario.ExpectedStateDelta.Txids, transactions.Txid{})
+				scenario.ExpectedStateDelta.Txids[innerAppCallTxn.Txn().ID()] = scenarioTxidValue
+			}
+			for i := range scenario.ExpectedEvents {
+				deltas := scenario.ExpectedEvents[i].Deltas
+				if deltas == nil {
+					continue
+				}
+				txidValue, ok := deltas.Txids[transactions.Txid{}]
+				if ok {
+					delete(deltas.Txids, transactions.Txid{})
+					deltas.Txids[innerAppCallTxn.Txn().ID()] = txidValue
+				}
+			}
+
 			txgroup := transactions.WrapSignedTxnsWithAD([]transactions.SignedTxn{
 				basicAppCallTxn.Txn().Sign(keys[0]),
 				payTxn.Txn().Sign(keys[1]),
-				innerAppCallTxn.Txn().Sign(keys[0]),
+				innerAppCallTxn.Txn().Sign(keys[4]),
 			})
 
 			require.Len(t, eval.block.Payset, 0)
@@ -388,7 +420,7 @@ int 1`,
 			}
 
 			expectedBasicAppCallAD := transactions.ApplyData{
-				ApplicationID: 1001,
+				ApplicationID: basicAppID,
 				EvalDelta: transactions.EvalDelta{
 					GlobalDelta: basics.StateDelta{},
 					LocalDeltas: map[uint64]basics.StateDelta{},
@@ -402,10 +434,107 @@ int 1`,
 					},
 				}
 
-			var expectedEvents = []mocktracer.Event{mocktracer.BeforeBlock(eval.block.Round())}
+			expectedFeeSinkData := ledgercore.ToAccountData(balances[testSinkAddr])
+			expectedFeeSinkData.MicroAlgos.Raw += txgroup[0].Txn.Fee.Raw
+			expectedAcct0Data := ledgercore.ToAccountData(balances[addrs[0]])
+			expectedAcct0Data.MicroAlgos.Raw -= txgroup[0].Txn.Fee.Raw
+			expectedAcct0Data.TotalAppParams = 1
+
+			expectedBlockHeader := eval.block.BlockHeader
+			expectedBasicAppCallDelta := ledgercore.StateDelta{
+				Accts: ledgercore.AccountDeltas{
+					Accts: []ledgercore.BalanceRecord{
+						{
+							Addr:        addrs[0],
+							AccountData: expectedAcct0Data,
+						},
+						{
+							Addr:        testSinkAddr,
+							AccountData: expectedFeeSinkData,
+						},
+					},
+					AppResources: []ledgercore.AppResourceRecord{
+						{
+							Aidx: basicAppID,
+							Addr: addrs[0],
+							Params: ledgercore.AppParamsDelta{
+								Params: &basics.AppParams{
+									ApprovalProgram:   txgroup[0].Txn.ApprovalProgram,
+									ClearStateProgram: txgroup[0].Txn.ClearStateProgram,
+								},
+							},
+						},
+					},
+				},
+				Creatables: map[basics.CreatableIndex]ledgercore.ModifiedCreatable{
+					basics.CreatableIndex(basicAppID): {
+						Ctype:   basics.AppCreatable,
+						Created: true,
+						Creator: addrs[0],
+					},
+				},
+				Txids: map[transactions.Txid]ledgercore.IncludedTransactions{
+					txgroup[0].Txn.ID(): {
+						LastValid: txgroup[0].Txn.LastValid,
+						Intra:     0,
+					},
+				},
+				Hdr:           &expectedBlockHeader,
+				PrevTimestamp: blkHeader.TimeStamp,
+			}
+			expectedBasicAppCallDelta.Hydrate()
+
+			expectedEvents := []mocktracer.Event{mocktracer.BeforeBlock(eval.block.Round())}
 			if testCase.firstTxnBehavior == "approve" {
 				err = eval.endOfBlock()
 				require.NoError(t, err)
+
+				expectedAcct1Data := ledgercore.AccountData{}
+				expectedAcct2Data := ledgercore.ToAccountData(balances[addrs[2]])
+				expectedAcct2Data.MicroAlgos.Raw += payTxn.Amount
+				expectedAcct3Data := ledgercore.ToAccountData(balances[addrs[3]])
+				expectedAcct3Data.MicroAlgos.Raw += expectedPayTxnAD.ClosingAmount.Raw
+				expectedFeeSinkData.MicroAlgos.Raw += txgroup[1].Txn.Fee.Raw
+
+				expectedPayTxnDelta := ledgercore.StateDelta{
+					Accts: ledgercore.AccountDeltas{
+						Accts: []ledgercore.BalanceRecord{
+							{
+								Addr:        addrs[1],
+								AccountData: expectedAcct1Data,
+							},
+							{
+								Addr:        testSinkAddr,
+								AccountData: expectedFeeSinkData,
+							},
+							{
+								Addr:        addrs[2],
+								AccountData: expectedAcct2Data,
+							},
+							{
+								Addr:        addrs[3],
+								AccountData: expectedAcct3Data,
+							},
+						},
+					},
+					Txids: map[transactions.Txid]ledgercore.IncludedTransactions{
+						txgroup[1].Txn.ID(): {
+							LastValid: txgroup[1].Txn.LastValid,
+							Intra:     0, // will be incremented once merged
+						},
+					},
+					Hdr:           &expectedBlockHeader,
+					PrevTimestamp: blkHeader.TimeStamp,
+				}
+				expectedPayTxnDelta.Hydrate()
+
+				expectedDelta := mocktracer.MergeStateDeltas(expectedBasicAppCallDelta, expectedPayTxnDelta, scenario.ExpectedStateDelta)
+
+				// If the scenario failed, we expect the failed txn ID to be removed from the group state delta
+				if scenario.Outcome != mocktracer.ApprovalOutcome {
+					delete(expectedDelta.Txids, txgroup[2].ID())
+				}
+
 				expectedEvents = append(expectedEvents, mocktracer.FlattenEvents([][]mocktracer.Event{
 					{
 						mocktracer.BeforeTxnGroup(3),
@@ -421,13 +550,14 @@ int 1`,
 					},
 					scenario.ExpectedEvents,
 					{
-						mocktracer.AfterTxnGroup(3, scenario.Outcome != mocktracer.ApprovalOutcome),
-					},
-					{
+						mocktracer.AfterTxnGroup(3, &expectedDelta, scenario.Outcome != mocktracer.ApprovalOutcome),
 						mocktracer.AfterBlock(eval.block.Round()),
 					},
 				})...)
 			} else {
+				// Removed failed txid from expected state delta
+				delete(expectedBasicAppCallDelta.Txids, txgroup[0].Txn.ID())
+
 				hasError := testCase.firstTxnBehavior == "error"
 				// EvalDeltas are removed from failed app call transactions
 				expectedBasicAppCallAD.EvalDelta = transactions.EvalDelta{}
@@ -441,11 +571,12 @@ int 1`,
 					{
 						mocktracer.AfterProgram(logic.ModeApp, hasError),
 						mocktracer.AfterTxn(protocol.ApplicationCallTx, expectedBasicAppCallAD, true), // end basicAppCallTxn
-						mocktracer.AfterTxnGroup(3, true),
+						mocktracer.AfterTxnGroup(3, &expectedBasicAppCallDelta, true),
 					},
 				})...)
 			}
-			require.Equal(t, expectedEvents, mocktracer.StripInnerTxnGroupIDsFromEvents(tracer.Events))
+			actualEvents := mocktracer.StripInnerTxnGroupIDsFromEvents(tracer.Events)
+			mocktracer.AssertEventsEqual(t, expectedEvents, actualEvents)
 		})
 	}
 }