[Test] - Nightly L1 Recovery

.github/workflows/nightly-l1-recovery.yml

@@ -0,0 +1,25 @@
+name: Nightly - L1 Recovery
+
+on:
+  schedule:
+    - cron: '30 3 * * *'
+  workflow_dispatch:
+
+jobs:
+  block-check:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+        with:
+          lfs: true
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Run Docker Compose
+        run: docker compose -f docker-compose-nightly.yml up --build --exit-code-from=block-checker

cmd/hack/rpc_cache/main.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/boltdb/bolt"
 	url2 "net/url"
+	"flag"
 )
 
 var db *bolt.DB
@@ -34,9 +35,9 @@ var paramsToExpire = map[string]struct{}{
 	"finalized": {},
 }
 
-func initDB() {
+func initDB(file string) {
 	var err error
-	db, err = bolt.Open("cache.db", 0600, nil)
+	db, err = bolt.Open(file, 0600, nil)
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -249,7 +250,10 @@ func handleCacheLookup(w http.ResponseWriter, r *http.Request) {
 }
 
 func main() {
-	initDB()
+	fileFlag := flag.String("file", "cache.db", "file to read")
+	flag.Parse()
+
+	initDB(*fileFlag)
 	defer db.Close()
 
 	http.HandleFunc("/", handleRequest)

eth/ethconfig/config_zkevm.go

@@ -46,6 +46,8 @@ type Zk struct {
 	MaxGasPrice                            uint64
 	GasPriceFactor                         float64
 	DAUrl                                  string
+	DataStreamHost                         string
+	DataStreamPort                         uint
 
 	RebuildTreeAfter    uint64
 	IncrementTreeAlways bool

turbo/cli/flags_zkevm.go

@@ -122,6 +122,8 @@ func ApplyFlagsForZkConfig(ctx *cli.Context, cfg *ethconfig.Config) {
 		DisableVirtualCounters:                 ctx.Bool(utils.DisableVirtualCounters.Name),
 		ExecutorPayloadOutput:                  ctx.String(utils.ExecutorPayloadOutput.Name),
 		DAUrl:                                  ctx.String(utils.DAUrl.Name),
+		DataStreamHost:                         ctx.String(utils.DataStreamHost.Name),
+		DataStreamPort:                         ctx.Uint(utils.DataStreamPort.Name),
 	}
 
 	checkFlag(utils.L2ChainIdFlag.Name, cfg.L2ChainId)
@@ -133,6 +135,8 @@ func ApplyFlagsForZkConfig(ctx *cli.Context, cfg *ethconfig.Config) {
 		checkFlag(utils.SequencerInitialForkId.Name, cfg.SequencerInitialForkId)
 		checkFlag(utils.ExecutorUrls.Name, cfg.ExecutorUrls)
 		checkFlag(utils.ExecutorStrictMode.Name, cfg.ExecutorStrictMode)
+		checkFlag(utils.DataStreamHost.Name, cfg.DataStreamHost)
+		checkFlag(utils.DataStreamPort.Name, cfg.DataStreamPort)
 
 		// if we are running in strict mode, the default, and we have no executor URLs then we panic
 		if cfg.ExecutorStrictMode && !cfg.HasExecutors() {

zk/debug_tools/nightly-block-compare-wait/main.go

@@ -0,0 +1,193 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"log"
+	"net/http"
+	"os"
+	"time"
+)
+
+const (
+	defaultCheckInterval = 30 * time.Second
+	defaultRunDuration   = 20 * time.Minute
+	defaultMaxBlockDiff  = 1000
+)
+
+type BlockNumberResponse struct {
+	Jsonrpc string `json:"jsonrpc"`
+	ID      int    `json:"id"`
+	Result  string `json:"result"`
+}
+
+type BlockResponse struct {
+	Jsonrpc string      `json:"jsonrpc"`
+	ID      int         `json:"id"`
+	Result  interface{} `json:"result"`
+	Error   interface{} `json:"error"`
+}
+
+func getBlockHeight(url string) (*BlockNumberResponse, error) {
+	reqBody := []byte(`{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":1}`)
+	req, err := http.NewRequest("POST", url, bytes.NewBuffer(reqBody))
+	if err != nil {
+		return nil, err
+	}
+
+	req.Header.Set("Content-Type", "application/json")
+
+	client := &http.Client{}
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("unexpected status code: %d", resp.StatusCode)
+	}
+
+	var jsonResp BlockNumberResponse
+	err = json.NewDecoder(resp.Body).Decode(&jsonResp)
+	if err != nil {
+		return nil, err
+	}
+
+	return &jsonResp, nil
+}
+
+func getBlockByNumber(url, blockNumber string) (*BlockResponse, error) {
+	reqBody := []byte(fmt.Sprintf(`{"jsonrpc":"2.0","method":"eth_getBlockByNumber","params":["%s", true],"id":1}`, blockNumber))
+	req, err := http.NewRequest("POST", url, bytes.NewBuffer(reqBody))
+	if err != nil {
+		return nil, err
+	}
+
+	req.Header.Set("Content-Type", "application/json")
+
+	client := &http.Client{}
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("unexpected status code: %d", resp.StatusCode)
+	}
+
+	var jsonResp BlockResponse
+	err = json.NewDecoder(resp.Body).Decode(&jsonResp)
+	if err != nil {
+		return nil, err
+	}
+
+	return &jsonResp, nil
+}
+
+func compareBlocks(erigonBlock, compareBlock *BlockResponse) bool {
+	if erigonBlock.Error != nil || compareBlock.Error != nil {
+		log.Println("Error in block responses:")
+		if erigonBlock.Error != nil {
+			log.Println("Erigon error:", erigonBlock.Error)
+			return false
+		}
+		if compareBlock.Error != nil {
+			log.Println("Compare node error:", compareBlock.Error)
+			return false
+		}
+	}
+
+	erigonJSON, _ := json.MarshalIndent(erigonBlock.Result, "", "  ")
+	compareJSON, _ := json.MarshalIndent(compareBlock.Result, "", "  ")
+
+	if bytes.Equal(erigonJSON, compareJSON) {
+		log.Println("Blocks match:", string(erigonJSON))
+		return true
+	} else {
+		log.Println("Blocks do not match:")
+		log.Println("Erigon block:", string(erigonJSON))
+		log.Println("Compare block:", string(compareJSON))
+		return false
+	}
+}
+
+func main() {
+	erigonNodeURL := flag.String("erigon", "http://erigon:8123", "URL for the Erigon node")
+	compareNodeURL := flag.String("compare", "http://other-node:8545", "URL for the compare node")
+	checkInterval := flag.Duration("interval", defaultCheckInterval, "Interval to check the block height")
+	runDuration := flag.Duration("duration", defaultRunDuration, "Total duration to run the checks")
+	maxBlockDiff := flag.Int("max-block-diff", defaultMaxBlockDiff, "Maximum allowed block difference")
+	flag.Parse()
+
+	ctx, cancel := context.WithTimeout(context.Background(), *runDuration)
+	defer cancel()
+
+	ticker := time.NewTicker(*checkInterval)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-ctx.Done():
+			log.Println("Stopping block height checks.")
+			os.Exit(0)
+		case <-ticker.C:
+			erigonHeight, err := getBlockHeight(*erigonNodeURL)
+			if err != nil {
+				log.Println("Error fetching block height from Erigon:", err)
+				continue
+			}
+
+			compareHeight, err := getBlockHeight(*compareNodeURL)
+			if err != nil {
+				log.Println("Error fetching block height from compare node:", err)
+				continue
+			}
+
+			erigonBlockNumber := erigonHeight.Result
+			compareBlockNumber := compareHeight.Result
+
+			if absDiff(erigonBlockNumber, compareBlockNumber) > *maxBlockDiff {
+				log.Println("Block heights are not within the allowed difference.")
+				continue
+			}
+
+			erigonBlock, err := getBlockByNumber(*erigonNodeURL, erigonBlockNumber)
+			if err != nil {
+				log.Println("Error fetching block from Erigon:", err)
+				continue
+			}
+
+			compareBlock, err := getBlockByNumber(*compareNodeURL, erigonBlockNumber)
+			if err != nil {
+				log.Println("Error fetching block from compare node:", err)
+				continue
+			}
+
+			ok := compareBlocks(erigonBlock, compareBlock)
+			if !ok {
+				// terminate on mismatch
+				log.Println("Block mismatch detected at height:", erigonBlock.Result.(map[string]interface{})["number"])
+				os.Exit(1)
+			}
+		}
+	}
+}
+
+func absDiff(a, b string) int {
+	var ia, ib int
+	fmt.Sscanf(a, "0x%x", &ia)
+	fmt.Sscanf(b, "0x%x", &ib)
+	return abs(ia - ib)
+}
+
+func abs(x int) int {
+	if x < 0 {
+		return -x
+	}
+	return x
+}

zk/tests/nightly-l1-recovery/docker-compose-nightly.yml

@@ -0,0 +1,44 @@
+version: '3.8'
+
+services:
+  cache:
+    image: golang:1.19
+    command: ["go", "run", "cmd/hack/rpc_cache/main.go", "-file", "zk/tests/nightly-l1-recovery/network5-cache.db"]
+    volumes:
+      - ../../../:/repo
+    working_dir: /repo
+    networks:
+      - erigon-net
+
+  erigon:
+    build:
+      context: ../../../
+      dockerfile: Dockerfile
+    command: ["--config", "/config/network5-config.yaml"]
+    environment:
+      - CDK_ERIGON_SEQUENCER=1
+    volumes:
+      - ./:/config
+      - datadir:/datadir
+    networks:
+      - erigon-net
+    depends_on:
+      - cache
+
+  block-checker:
+    image: golang:1.19
+    command: ["go", "run", "/repo/zk/debug_tools/nightly-block-compare-wait/main.go", "--compare=http://34.175.214.161:8505"]
+    volumes:
+      - ../../../:/repo
+    working_dir: /repo
+    networks:
+      - erigon-net
+    depends_on:
+      - erigon
+
+networks:
+  erigon-net:
+    driver: bridge
+
+volumes:
+  datadir:

zk/tests/nightly-l1-recovery/dynamic-integration-chainspec.json

@@ -0,0 +1,24 @@
+{
+  "ChainName": "dynamic-integration",
+  "chainId": 8282,
+  "consensus": "ethash",
+  "homesteadBlock": 0,
+  "daoForkBlock": 0,
+  "eip150Block": 0,
+  "eip155Block": 0,
+  "byzantiumBlock": 0,
+  "constantinopleBlock": 0,
+  "petersburgBlock": 0,
+  "istanbulBlock": 0,
+  "muirGlacierBlock": 0,
+  "berlinBlock": 0,
+  "londonBlock": 9999999999999999999999999999999999999999999999999,
+  "arrowGlacierBlock": 9999999999999999999999999999999999999999999999999,
+  "grayGlacierBlock": 9999999999999999999999999999999999999999999999999,
+  "terminalTotalDifficulty": 58750000000000000000000,
+  "terminalTotalDifficultyPassed": false,
+  "shanghaiTime": 9999999999999999999999999999999999999999999999999,
+  "cancunTime": 9999999999999999999999999999999999999999999999999,
+  "pragueTime": 9999999999999999999999999999999999999999999999999,
+  "ethash": {}
+}

zk/tests/nightly-l1-recovery/dynamic-integration-conf.json

@@ -0,0 +1,6 @@
+{
+  "root": "0xcea48f8380bdc5766adab682d3c9dcf9ded87274dc38d0f81e96a7679a6ca7f0",
+  "timestamp": 1717430388,
+  "gasLimit": 0,
+  "difficulty": 0
+}