[1.0] Improve restart production integration test

tests/TestHarness/Node.py

@@ -541,6 +541,9 @@ def modifyBuiltinPFSubjRestrictions(self, featureCodename, subjectiveRestriction
         with open(jsonPath, "w") as f:
             json.dump(protocolFeatureJson, f, indent=2)
 
+    def getFinalizerInfo(self):
+       return self.processUrllibRequest("chain", "get_finalizer_info",silentErrors=False, exitOnError=True)
+
     # Require producer_api_plugin
     def createSnapshot(self):
         return self.processUrllibRequest("producer", "create_snapshot")

tests/production_restart.py

@@ -18,7 +18,7 @@
 # Once everything has been confirmed to be working correctly and finality is advancing, cleanly shut down the producer
 # defproducera node but keep the finalizer node of defproducerb running.
 #
-# Then change the finalizer policy (e.g. switch the order of the two finalizers) to get the nodes into a state where
+# Then change the finalizer policy (e.g. replace a key in node defproducera) to get the nodes into a state where
 # they have a pending finalizer policy. At that point restart the producer node defproducera (with stale production
 # enabled so it produces blocks again).
 #
@@ -64,28 +64,54 @@
     cluster.biosNode.kill(signal.SIGTERM)
     cluster.waitOnClusterSync(blockAdvancing=5)
 
-    node0 = cluster.getNode(0) # producer
-    node1 = cluster.getNode(1) # finalizer
+    producerNode = cluster.getNode(0)
+    finalizerNode = cluster.getNode(1)
 
     Print("Wait for lib to advance")
-    assert node1.waitForLibToAdvance(), "Node1 did not advance LIB"
-    assert node0.waitForLibToAdvance(), "Node0 did not advance LIB"
+    assert finalizerNode.waitForLibToAdvance(), "finalizerNode did not advance LIB"
+    assert producerNode.waitForLibToAdvance(), "producerNode did not advance LIB"
 
     Print("Set finalizers so a pending is in play")
-    assert cluster.setFinalizers([node1, node0], node0), "setfinalizers failed" # switch order
-    assert node0.waitForLibToAdvance(), "Node0 did not advance LIB after setfinalizers"
-    node0.waitForHeadToAdvance() # get additional qc
-
-    Print("Shutdown producer node0")
-    node0.kill(signal.SIGTERM)
-    assert not node0.verifyAlive(), "Node0 did not shutdown"
-
-    Print("Restart producer node0")
-    node0.relaunch(chainArg=" -e ")
+    # Try a number of times to make sure we have a pending policy
+    numTrys = 10
+    for i in range(0, numTrys):
+       # Switch BLS keys on producerNode. setFinalizers uses the first key
+       # in the configured key list (index 0)
+       newBlsPubKey  = producerNode.keys[1].blspubkey
+       newBlsPrivKey = producerNode.keys[1].blsprivkey
+       newBlsPop     = producerNode.keys[1].blspop
+       producerNode.keys[1].blspubkey  = producerNode.keys[0].blspubkey
+       producerNode.keys[1].blsprivkey = producerNode.keys[0].blsprivkey
+       producerNode.keys[1].blspop     = producerNode.keys[0].blspop
+       producerNode.keys[0].blspubkey  = newBlsPubKey
+       producerNode.keys[0].blsprivkey = newBlsPrivKey
+       producerNode.keys[0].blspop     = newBlsPop
+
+       assert cluster.setFinalizers([producerNode, finalizerNode], producerNode), "setfinalizers failed"
+       assert producerNode.waitForLibToAdvance(), "producerNode did not advance LIB after setfinalizers"
+       producerNode.waitForHeadToAdvance() # get additional qc
+
+       # Check if a pending policy exists
+       finalizerInfo = producerNode.getFinalizerInfo()
+       Print(f"{finalizerInfo}")
+       if finalizerInfo["payload"]["pending_finalizer_policy"] is not None and finalizerInfo["payload"]["pending_finalizer_policy"]["finalizers"][0]["public_key"] == newBlsPubKey:
+          Print(f"Got a pending policy in {i+1} attempts")
+          break
+       else:
+          Print(f"Trying to get a pending policy, {i+1} attempts")
+
+    # It is OK if pending policy does not exist. During manual tests,
+    # pending policy always exists at the first try
+    Print("Shutdown producer producerNode")
+    producerNode.kill(signal.SIGTERM)
+    assert not producerNode.verifyAlive(), "producerNode did not shutdown"
+
+    Print("Restart producer producerNode")
+    producerNode.relaunch(chainArg=" -e ")
 
     Print("Verify LIB advances after restart")
-    assert node0.waitForLibToAdvance(), "Node0 did not advance LIB"
-    assert node1.waitForLibToAdvance(), "Node1 did not advance LIB"
+    assert producerNode.waitForLibToAdvance(), "producerNode did not advance LIB"
+    assert finalizerNode.waitForLibToAdvance(), "finalizerNode did not advance LIB"
 
     testSuccessful=True
 finally:

tests/production_restart_test_shape.json

@@ -30,6 +30,13 @@
             "blspubkey":"PUB_BLS_rYRa_-bT7uLOSAfPIBy6NlXFB0YxwROeSuqHzw6s-1cuK_-GJUKqp20ktyAnsO4ZuHdx3BEPDaLronpnL22MXKWM7bvZnkCfbGCD6OzizQqxXkM9N5z5R-OUA4Ime6cF5YTSFg",
             "blsprivkey":"PVT_BLS_GQjR0E8Hu8KrsTCvLKnlOCIwQijAj2-5KDizQwF-bAY6pise",
             "blspop":"SIG_BLS_syFMuifUnX2zQQKr0cuHYzQQjsuPrNG75_z6y8fOyYg_twqMICZ0kT7ObbwIOUsLfXx9PVb4-QLEgUYGSRg1NSfeHGjIGkhea82wa3ayfI8elUEU1MStKbeKpys7xUAQz1PEgwcz5dClq3HyLQmMAjpoL74N_Znf0KiNEVZMte-DLF7x_6sAfp_834LthyYHjZYTmdG7belyzlYHKJb6upnZy9nR_zoKpx9jeTd3tzVhoTCuAN6aFw68D_ItY5cWiY2dhA"
+          },
+          {
+            "pubkey": "EOS8XH2gKxsef9zxmMHm4vaSvxQUhg7W4GC3nK2KSRxyYrNG5gZFS",
+            "privkey": "5JcoRRhDcgm51dkBrRTmErceTqrYhrq22UnmUjTZToMpH91B9N1",
+            "blspubkey":"PUB_BLS_JzblSr2sf_UhxQjGxOtHbRCBkHgSB1RG4xUbKKl-fKtUjx6hyOHajnVQT4IvBF4PutlX7JTC14IqIjADlP-3_G2MXRhBlkB57r2u59OCwRQQEDqmVSADf6CoT8zFUXcSgHFw7w",
+            "blsprivkey":"PVT_BLS_QRxLAVbe2n7RaPWx2wHbur8erqUlAs-V_wXasGhjEA78KlBq",
+            "blspop":"SIG_BLS_Z5fJqFv6DIsHFhBFpkHmL_R48h80zVKQHtB5lrKGOVZTaSQNuVaXD_eHg7HBvKwY6zqgA_vryCLQo5W0Inu6HtLkGL2gYX2UHJjrZJZpfJSKG0ynqAZmyrCglxRLNm8KkFdGGR8oJXf5Yzyu7oautqTPniuKLBvNeQxGJGDOQtHSQ0uP3mD41pWzPFRoi10BUor9MbwUTQ7fO7Of4ZjhVM3IK4JrqX1RBXkDX83Wi9xFzs_fdPIyMqmgEzFgolgUa8XN4Q"
           }
         ],
         "peers": [