Merge pull request #2696 from m-Peter/cadence-testing-framework-impro…

…vements

Cadence testing framework improvements

runtime/stdlib/contracts/test.cdc

@@ -122,10 +122,10 @@ pub contract Test {
             return self.backend.events(type)
         }
 
-        /// Resets the state of the blockchain.
+        /// Resets the state of the blockchain to the given height.
         ///
-        pub fun reset() {
-            self.backend.reset()
+        pub fun reset(to height: UInt64) {
+            self.backend.reset(to: height)
         }
     }
 
@@ -168,9 +168,13 @@ pub contract Test {
     /// operations, such as transactions and scripts.
     ///
     pub struct interface Result {
-        /// The resulted status of an executed operation.
+        /// The result status of an executed operation.
         ///
         pub let status: ResultStatus
+
+        /// The optional error of an executed operation.
+        ///
+        pub let error: Error?
     }
 
     /// The result of a transaction execution.
@@ -305,9 +309,9 @@ pub contract Test {
         ///
         pub fun events(_ type: Type?): [AnyStruct]
 
-        /// Resets the state of the blockchain.
+        /// Resets the state of the blockchain to the given height.
         ///
-        pub fun reset()
+        pub fun reset(to height: UInt64)
     }
 
     /// Returns a new matcher that negates the test of the given matcher.
@@ -346,4 +350,28 @@ pub contract Test {
         })
     }
 
+    /// Asserts that the result status of an executed operation, such as
+    /// a script or transaction, has failed and contains the given error
+    /// message.
+    ///
+    pub fun assertError(_ result: {Result}, errorMessage: String) {
+        pre {
+            result.status == ResultStatus.failed: "no error was found"
+        }
+
+        var found = false
+        let msg = result.error!.message
+        let msgLength = msg.length - errorMessage.length + 1
+        var i = 0
+        while i < msgLength {
+            if msg.slice(from: i, upTo: i + errorMessage.length) == errorMessage {
+                found = true
+                break
+            }
+            i = i + 1
+        }
+
+        assert(found, message: "the error message did not contain the given sub-string")
+    }
+
 }

runtime/stdlib/test-framework.go

@@ -70,7 +70,7 @@ type TestFramework interface {
 		eventType interpreter.StaticType,
 	) interpreter.Value
 
-	Reset()
+	Reset(uint64)
 }
 
 type ScriptResult struct {

runtime/stdlib/test_emulatorbackend.go

@@ -666,16 +666,20 @@ func (t *testEmulatorBackendType) newEventsFunction(
 const testEmulatorBackendTypeResetFunctionName = "reset"
 
 const testEmulatorBackendTypeResetFunctionDocString = `
-Resets the state of the blockchain.
+Resets the state of the blockchain to the given height.
 `
 
 func (t *testEmulatorBackendType) newResetFunction(
 	testFramework TestFramework,
 ) *interpreter.HostFunctionValue {
 	return interpreter.NewUnmeteredHostFunctionValue(
-		t.eventsFunctionType,
+		t.resetFunctionType,
 		func(invocation interpreter.Invocation) interpreter.Value {
-			testFramework.Reset()
+			height, ok := invocation.Arguments[0].(interpreter.UInt64Value)
+			if !ok {
+				panic(errors.NewUnreachableError())
+			}
+			testFramework.Reset(uint64(height))
 			return interpreter.Void
 		},
 	)

runtime/stdlib/test_test.go

@@ -1154,6 +1154,112 @@ func TestTestBeFailedMatcher(t *testing.T) {
 	})
 }
 
+func TestTestAssertErrorMatcher(t *testing.T) {
+
+	t.Parallel()
+
+	t.Run("with ScriptResult", func(t *testing.T) {
+		t.Parallel()
+
+		const script = `
+            import Test
+
+            pub fun testMatch() {
+                let result = Test.ScriptResult(
+                    status: Test.ResultStatus.failed,
+                    returnValue: nil,
+                    error: Test.Error("computation exceeding limit")
+                )
+
+                Test.assertError(result, errorMessage: "exceeding limit")
+            }
+
+            pub fun testNoMatch() {
+                let result = Test.ScriptResult(
+                    status: Test.ResultStatus.failed,
+                    returnValue: nil,
+                    error: Test.Error("computation exceeding memory")
+                )
+
+                Test.assertError(result, errorMessage: "exceeding limit")
+            }
+
+            pub fun testNoError() {
+                let result = Test.ScriptResult(
+                    status: Test.ResultStatus.succeeded,
+                    returnValue: 42,
+                    error: nil
+                )
+
+                Test.assertError(result, errorMessage: "exceeding limit")
+            }
+		`
+
+		inter, err := newTestContractInterpreter(t, script)
+		require.NoError(t, err)
+
+		_, err = inter.Invoke("testMatch")
+		require.NoError(t, err)
+
+		_, err = inter.Invoke("testNoMatch")
+		require.Error(t, err)
+		assert.ErrorContains(t, err, "the error message did not contain the given sub-string")
+
+		_, err = inter.Invoke("testNoError")
+		require.Error(t, err)
+		assert.ErrorContains(t, err, "no error was found")
+	})
+
+	t.Run("with TransactionResult", func(t *testing.T) {
+		t.Parallel()
+
+		const script = `
+            import Test
+
+            pub fun testMatch() {
+                let result = Test.TransactionResult(
+                    status: Test.ResultStatus.failed,
+                    error: Test.Error("computation exceeding limit")
+                )
+
+                Test.assertError(result, errorMessage: "exceeding limit")
+            }
+
+            pub fun testNoMatch() {
+                let result = Test.TransactionResult(
+                    status: Test.ResultStatus.failed,
+                    error: Test.Error("computation exceeding memory")
+                )
+
+                Test.assertError(result, errorMessage: "exceeding limit")
+            }
+
+            pub fun testNoError() {
+                let result = Test.TransactionResult(
+                    status: Test.ResultStatus.succeeded,
+                    error: nil
+                )
+
+                Test.assertError(result, errorMessage: "exceeding limit")
+            }
+		`
+
+		inter, err := newTestContractInterpreter(t, script)
+		require.NoError(t, err)
+
+		_, err = inter.Invoke("testMatch")
+		require.NoError(t, err)
+
+		_, err = inter.Invoke("testNoMatch")
+		require.Error(t, err)
+		assert.ErrorContains(t, err, "the error message did not contain the given sub-string")
+
+		_, err = inter.Invoke("testNoError")
+		require.Error(t, err)
+		assert.ErrorContains(t, err, "no error was found")
+	})
+}
+
 func TestTestBeNilMatcher(t *testing.T) {
 
 	t.Parallel()
@@ -2023,6 +2129,62 @@ func TestBlockchain(t *testing.T) {
 		assert.True(t, eventsInvoked)
 	})
 
+	t.Run("reset", func(t *testing.T) {
+		t.Parallel()
+
+		const script = `
+            import Test
+
+            pub fun test() {
+                let blockchain = Test.newEmulatorBlockchain()
+                blockchain.reset(to: 5)
+            }
+		`
+
+		resetInvoked := false
+
+		testFramework := &mockedTestFramework{
+			reset: func(height uint64) {
+				resetInvoked = true
+				assert.Equal(t, uint64(5), height)
+			},
+		}
+
+		inter, err := newTestContractInterpreterWithTestFramework(t, script, testFramework)
+		require.NoError(t, err)
+
+		_, err = inter.Invoke("test")
+		require.NoError(t, err)
+
+		assert.True(t, resetInvoked)
+	})
+
+	t.Run("reset with type mismatch for height", func(t *testing.T) {
+		t.Parallel()
+
+		const script = `
+            import Test
+
+            pub fun test() {
+                let blockchain = Test.newEmulatorBlockchain()
+                blockchain.reset(to: 5.5)
+            }
+		`
+
+		resetInvoked := false
+
+		testFramework := &mockedTestFramework{
+			reset: func(height uint64) {
+				resetInvoked = true
+			},
+		}
+
+		_, err := newTestContractInterpreterWithTestFramework(t, script, testFramework)
+		errs := checker.RequireCheckerErrors(t, err, 1)
+		assert.IsType(t, &sema.TypeMismatchError{}, errs[0])
+		assert.False(t, resetInvoked)
+	})
+
 	// TODO: Add more tests for the remaining functions.
 }
 
@@ -2039,7 +2201,7 @@ type mockedTestFramework struct {
 	logs               func() []string
 	serviceAccount     func() (*Account, error)
 	events             func(inter *interpreter.Interpreter, eventType interpreter.StaticType) interpreter.Value
-	reset              func()
+	reset              func(uint64)
 }
 
 var _ TestFramework = &mockedTestFramework{}
@@ -2159,10 +2321,10 @@ func (m mockedTestFramework) Events(
 	return m.events(inter, eventType)
 }
 
-func (m mockedTestFramework) Reset() {
+func (m mockedTestFramework) Reset(height uint64) {
 	if m.reset == nil {
 		panic("'Reset' is not implemented")
 	}
 
-	m.reset()
+	m.reset(height)
 }