Merge branch 'master' into remove_fast_config_

.github/workflows/ci.yml

@@ -138,6 +138,7 @@ jobs:
           echo "TMPDIR=$(pwd)/target/tmp/deadbeefbee" >> "$GITHUB_ENV"
           echo "GOMEMLIMIT=6GiB" >> "$GITHUB_ENV"
           echo "GOGC=80" >> "$GITHUB_ENV"
+          echo "GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }}" >> "$GITHUB_ENV"
 
       - name: run tests without race detection and path state scheme
         if: matrix.test-mode == 'defaults'

Makefile

@@ -147,78 +147,119 @@ stylus_benchmarks = $(wildcard $(stylus_dir)/*.toml $(stylus_dir)/src/*.rs) $(st
 
 # user targets
 
+.PHONY: push
 push: lint test-go .make/fmt
 	@printf "%bdone building %s%b\n" $(color_pink) $$(expr $$(echo $? | wc -w) - 1) $(color_reset)
 	@printf "%bready for push!%b\n" $(color_pink) $(color_reset)
 
+.PHONY: all
 all: build build-replay-env test-gen-proofs
 	@touch .make/all
 
+.PHONY: build
 build: $(patsubst %,$(output_root)/bin/%, nitro deploy relay daserver datool seq-coordinator-invalidate nitro-val seq-coordinator-manager)
 	@printf $(done)
 
+.PHONY: build-node-deps
 build-node-deps: $(go_source) build-prover-header build-prover-lib build-jit .make/solgen .make/cbrotli-lib
 
+.PHONY: test-go-deps
 test-go-deps: \
 	build-replay-env \
 	$(stylus_test_wasms) \
 	$(arbitrator_stylus_lib) \
 	$(arbitrator_generated_header) \
 	$(patsubst %,$(arbitrator_cases)/%.wasm, global-state read-inboxmsg-10 global-state-wrapper const)
 
+.PHONY: build-prover-header
 build-prover-header: $(arbitrator_generated_header)
 
+.PHONY: build-prover-lib
 build-prover-lib: $(arbitrator_stylus_lib)
 
+.PHONY: build-prover-bin
 build-prover-bin: $(prover_bin)
 
+.PHONY: build-jit
 build-jit: $(arbitrator_jit)
 
+.PHONY: build-replay-env
 build-replay-env: $(prover_bin) $(arbitrator_jit) $(arbitrator_wasm_libs) $(replay_wasm) $(output_latest)/machine.wavm.br
 
+.PHONY: build-wasm-libs
 build-wasm-libs: $(arbitrator_wasm_libs)
 
+.PHONY: build-wasm-bin
 build-wasm-bin: $(replay_wasm)
 
+.PHONY: build-solidity
 build-solidity: .make/solidity
 
+.PHONY: contracts
 contracts: .make/solgen
 	@printf $(done)
 
+.PHONY: format fmt
 format fmt: .make/fmt
 	@printf $(done)
 
+.PHONY: lint
 lint: .make/lint
 	@printf $(done)
 
+.PHONY: stylus-benchmarks
 stylus-benchmarks: $(stylus_benchmarks)
 	cargo test --manifest-path $< --release --features benchmark benchmark_ -- --nocapture
 	@printf $(done)
 
+.PHONY: test-go
 test-go: .make/test-go
 	@printf $(done)
 
+.PHONY: test-go-challenge
 test-go-challenge: test-go-deps
-	go test -v -timeout 120m ./system_tests/... -run TestChallenge -tags challengetest
+	gotestsum --format short-verbose --no-color=false -- -timeout 120m ./system_tests/... -run TestChallenge -tags challengetest
 	@printf $(done)
 
+.PHONY: test-go-stylus
 test-go-stylus: test-go-deps
-	go test -v -timeout 120m ./system_tests/... -run TestProgramArbitrator -tags stylustest
+	gotestsum --format short-verbose --no-color=false -- -timeout 120m ./system_tests/... -run TestProgramArbitrator -tags stylustest
 	@printf $(done)
 
+.PHONY: test-go-redis
 test-go-redis: test-go-deps
-	TEST_REDIS=redis://localhost:6379/0 go test -p 1 -run TestRedis ./system_tests/... ./arbnode/...
+	TEST_REDIS=redis://localhost:6379/0 gotestsum --format short-verbose --no-color=false -- -p 1 -run TestRedis ./system_tests/... ./arbnode/...
 	@printf $(done)
 
+.PHONY: test-gen-proofs
 test-gen-proofs: \
         $(arbitrator_test_wasms) \
 	$(patsubst $(arbitrator_cases)/%.wat,contracts/test/prover/proofs/%.json, $(arbitrator_tests_wat)) \
 	$(patsubst $(arbitrator_cases)/rust/src/bin/%.rs,contracts/test/prover/proofs/rust-%.json, $(arbitrator_tests_rust)) \
 	contracts/test/prover/proofs/go.json
+	@printf $(done)
+
+.PHONY: test-rust
+test-rust: .make/test-rust
+	@printf $(done)
+
+# Runs the fastest and most reliable and high-value tests.
+.PHONY: tests
+tests: test-go test-rust
+	@printf $(done)
 
+# Runs all tests, including slow and unreliable tests.
+#  Currently, NOT including:
+#  - test-go-redis (These testts require additional setup and are not as reliable)
+.PHONY: tests-all
+tests-all: tests test-go-challenge test-go-stylus test-gen-proofs
+	@printf $(done)
+
+.PHONY: wasm-ci-build
 wasm-ci-build: $(arbitrator_wasm_libs) $(arbitrator_test_wasms) $(stylus_test_wasms) $(output_latest)/user_test.wasm
 	@printf $(done)
 
+.PHONY: clean
 clean:
 	go clean -testcache
 	rm -rf $(arbitrator_cases)/rust/target
@@ -237,6 +278,7 @@ clean:
 	@rm -rf contracts/build contracts/cache solgen/go/
 	@rm -f .make/*
 
+.PHONY: docker
 docker:
 	docker build -t nitro-node-slim --target nitro-node-slim .
 	docker build -t nitro-node --target nitro-node .
@@ -490,6 +532,10 @@ contracts/test/prover/proofs/%.json: $(arbitrator_cases)/%.wasm $(prover_bin)
 	gotestsum --format short-verbose --no-color=false
 	@touch $@
 
+.make/test-rust: $(DEP_PREDICATE) wasm-ci-build $(ORDER_ONLY_PREDICATE) .make
+	cargo test --manifest-path arbitrator/Cargo.toml --release
+	@touch $@
+
 .make/solgen: $(DEP_PREDICATE) solgen/gen.go .make/solidity $(ORDER_ONLY_PREDICATE) .make
 	mkdir -p solgen/go/
 	go run solgen/gen.go
@@ -537,4 +583,3 @@ contracts/test/prover/proofs/%.json: $(arbitrator_cases)/%.wasm $(prover_bin)
 
 always:              # use this to force other rules to always build
 .DELETE_ON_ERROR:    # causes a failure to delete its target
-.PHONY: push all build build-node-deps test-go-deps build-prover-header build-prover-lib build-prover-bin build-jit build-replay-env build-solidity build-wasm-libs contracts format fmt lint stylus-benchmarks test-go test-gen-proofs push clean docker

cmd/nitro-val/nitro_val.go

@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"fmt"
+	"math"
 	_ "net/http/pprof" // #nosec G108
 	"os"
 	"os/signal"
@@ -66,6 +67,8 @@ func mainImpl() int {
 	}
 	stackConf := DefaultValidationNodeStackConfig
 	stackConf.DataDir = "" // ephemeral
+	stackConf.HTTPBodyLimit = math.MaxInt
+	stackConf.WSReadLimit = math.MaxInt64
 	nodeConfig.HTTP.Apply(&stackConf)
 	nodeConfig.WS.Apply(&stackConf)
 	nodeConfig.Auth.Apply(&stackConf)

go.mod

@@ -28,6 +28,7 @@ require (
 	github.com/gobwas/httphead v0.1.0
 	github.com/gobwas/ws v1.2.1
 	github.com/gobwas/ws-examples v0.0.0-20190625122829-a9e8908d9484
+	github.com/google/btree v1.1.2
 	github.com/google/go-cmp v0.6.0
 	github.com/google/uuid v1.3.0
 	github.com/hashicorp/golang-lru/v2 v2.0.7
@@ -50,10 +51,7 @@ require (
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0
 )
 
-require (
-	github.com/google/btree v1.1.2 // indirect
-	github.com/google/go-querystring v1.1.0 // indirect
-)
+require github.com/google/go-querystring v1.1.0 // indirect
 
 require (
 	github.com/DataDog/zstd v1.4.5 // indirect
@@ -165,6 +163,7 @@ require (
 	go.opencensus.io v0.22.5 // indirect
 	golang.org/x/mod v0.14.0 // indirect
 	golang.org/x/net v0.23.0 // indirect
+	golang.org/x/oauth2 v0.22.0
 	golang.org/x/sync v0.5.0
 	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/time v0.3.0 // indirect

go.sum

@@ -340,7 +340,6 @@ github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb h1:PBC98N2aIaM3XXi
 github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/gomodule/redigo v1.7.1-0.20190724094224-574c33c3df38/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
-github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU=
 github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4=
@@ -839,6 +838,8 @@ golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4Iltr
 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
+golang.org/x/oauth2 v0.22.0 h1:BzDx2FehcG7jJwgWLELCdmLuxk2i+x9UDpSiss2u0ZA=
+golang.org/x/oauth2 v0.22.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=

scripts/split-val-entry.sh

@@ -16,4 +16,4 @@ for port in 52000 52001; do
     done
 done
 echo launching nitro-node
-/usr/local/bin/nitro --validation.wasm.allowed-wasm-module-roots /home/user/nitro-legacy/machines,/home/user/target/machines --node.block-validator.validation-server-configs-list='[{"jwtsecret":"/tmp/nitro-val.jwt","url":"http://127.0.0.10:52000"}, {"jwtsecret":"/tmp/nitro-val.jwt","url":"http://127.0.0.10:52001"}]' "$@"
+/usr/local/bin/nitro --validation.wasm.allowed-wasm-module-roots /home/user/nitro-legacy/machines,/home/user/target/machines --node.block-validator.validation-server-configs-list='[{"jwtsecret":"/tmp/nitro-val.jwt","url":"ws://127.0.0.10:52000"}, {"jwtsecret":"/tmp/nitro-val.jwt","url":"ws://127.0.0.10:52001"}]' "$@"

staker/block_validator.go

@@ -8,6 +8,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"net/url"
 	"regexp"
 	"runtime"
 	"sync"
@@ -140,6 +141,16 @@ func (c *BlockValidatorConfig) Validate() error {
 		if err := c.ValidationServerConfigs[i].Validate(); err != nil {
 			return fmt.Errorf("failed to validate one of the block-validator validation-server-configs. url: %s, err: %w", c.ValidationServerConfigs[i].URL, err)
 		}
+		serverUrl := c.ValidationServerConfigs[i].URL
+		if len(serverUrl) > 0 && serverUrl != "self" && serverUrl != "self-auth" {
+			u, err := url.Parse(serverUrl)
+			if err != nil {
+				return fmt.Errorf("failed parsing validation server's url:%s err: %w", serverUrl, err)
+			}
+			if u.Scheme != "ws" && u.Scheme != "wss" {
+				return fmt.Errorf("validation server's url scheme is unsupported, it should either be ws or wss, url:%s", serverUrl)
+			}
+		}
 	}
 	return nil
 }

staker/stateless_block_validator.go

@@ -7,7 +7,6 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"net/url"
 	"runtime"
 	"testing"
 
@@ -488,13 +487,8 @@ func (v *StatelessBlockValidator) Start(ctx_in context.Context) error {
 			return fmt.Errorf("starting execution spawner: %w", err)
 		}
 	}
-	for i, spawner := range v.execSpawners {
+	for _, spawner := range v.execSpawners {
 		if err := spawner.Start(ctx_in); err != nil {
-			if u, parseErr := url.Parse(v.config.ValidationServerConfigs[i].URL); parseErr == nil {
-				if u.Scheme != "ws" && u.Scheme != "wss" {
-					return fmt.Errorf("validation server's url scheme is unsupported, it should either be ws or wss, url:%s err: %w", v.config.ValidationServerConfigs[i].URL, err)
-				}
-			}
 			return err
 		}
 	}

system_tests/block_validator_test.go

@@ -92,7 +92,7 @@ func testBlockValidatorSimple(t *testing.T, opts Options) {
 		validatorConfig.BlockValidator.RedisValidationClientConfig = redis.ValidationClientConfig{}
 	}
 
-	AddDefaultValNode(t, ctx, validatorConfig, !opts.arbitrator, redisURL, opts.wasmRootDir)
+	AddValNode(t, ctx, validatorConfig, !opts.arbitrator, redisURL, opts.wasmRootDir)
 
 	testClientB, cleanupB := builder.Build2ndNode(t, &SecondNodeParams{nodeConfig: validatorConfig})
 	defer cleanupB()

system_tests/common_test.go

@@ -364,7 +364,7 @@ func (b *NodeBuilder) BuildL2OnL1(t *testing.T) func() {
 		validatorTxOptsPtr = &validatorTxOpts
 	}
 
-	AddDefaultValNode(t, b.ctx, b.nodeConfig, true, "", b.valnodeConfig.Wasm.RootPath)
+	AddValNodeIfNeeded(t, b.ctx, b.nodeConfig, true, "", b.valnodeConfig.Wasm.RootPath)
 
 	Require(t, b.execConfig.Validate())
 	execConfig := b.execConfig
@@ -400,7 +400,7 @@ func (b *NodeBuilder) BuildL2OnL1(t *testing.T) func() {
 func (b *NodeBuilder) BuildL2(t *testing.T) func() {
 	b.L2 = NewTestClient(b.ctx)
 
-	AddDefaultValNode(t, b.ctx, b.nodeConfig, true, "", b.valnodeConfig.Wasm.RootPath)
+	AddValNodeIfNeeded(t, b.ctx, b.nodeConfig, true, "", b.valnodeConfig.Wasm.RootPath)
 
 	var chainDb ethdb.Database
 	var arbDb ethdb.Database
@@ -891,10 +891,14 @@ func currentRootModule(t *testing.T) common.Hash {
 	return locator.LatestWasmModuleRoot()
 }
 
-func AddDefaultValNode(t *testing.T, ctx context.Context, nodeConfig *arbnode.Config, useJit bool, redisURL string, wasmRootDir string) {
-	if !nodeConfig.ValidatorRequired() {
+func AddValNodeIfNeeded(t *testing.T, ctx context.Context, nodeConfig *arbnode.Config, useJit bool, redisURL string, wasmRootDir string) {
+	if !nodeConfig.ValidatorRequired() || nodeConfig.BlockValidator.ValidationServerConfigs[0].URL != "" {
 		return
 	}
+	AddValNode(t, ctx, nodeConfig, useJit, redisURL, wasmRootDir)
+}
+
+func AddValNode(t *testing.T, ctx context.Context, nodeConfig *arbnode.Config, useJit bool, redisURL string, wasmRootDir string) {
 	conf := valnode.TestValidationConfig
 	conf.UseJit = useJit
 	conf.Wasm.RootPath = wasmRootDir
@@ -1168,7 +1172,7 @@ func Create2ndNodeWithConfig(
 	l2blockchain, err := gethexec.WriteOrTestBlockChain(l2chainDb, coreCacheConfig, initReader, chainConfig, initMessage, ExecConfigDefaultTest().TxLookupLimit, 0)
 	Require(t, err)
 
-	AddDefaultValNode(t, ctx, nodeConfig, true, "", valnodeConfig.Wasm.RootPath)
+	AddValNodeIfNeeded(t, ctx, nodeConfig, true, "", valnodeConfig.Wasm.RootPath)
 
 	Require(t, execConfig.Validate())
 	Require(t, nodeConfig.Validate())

system_tests/program_recursive_test.go

@@ -136,7 +136,7 @@ func testProgramResursiveCalls(t *testing.T, tests [][]multiCallRecurse, jit boo
 	validatorConfig.BlockValidator.Enable = true
 	emptyRedisURL := ""
 	defaultWasmRootPath := ""
-	AddDefaultValNode(t, ctx, validatorConfig, jit, emptyRedisURL, defaultWasmRootPath)
+	AddValNode(t, ctx, validatorConfig, jit, emptyRedisURL, defaultWasmRootPath)
 	valClient, valCleanup := builder.Build2ndNode(t, &SecondNodeParams{nodeConfig: validatorConfig})
 	defer valCleanup()
 

util/testhelpers/github/releases.go

@@ -4,10 +4,12 @@ import (
 	"context"
 	"fmt"
 	"net/url"
+	"os"
 	"regexp"
 	"strings"
 
 	"github.com/google/go-github/v62/github"
+	"golang.org/x/oauth2"
 )
 
 var wasmRootExp = regexp.MustCompile(`\*\*WAVM Module Root\*\*: (0x[a-f0-9]{64})`)
@@ -18,9 +20,18 @@ type ConsensusRelease struct {
 	ReplayWasmURL  url.URL
 }
 
+func getAuthGitClient(ctx context.Context) *github.Client {
+	token := os.Getenv("GITHUB_TOKEN")
+	if token == "" {
+		return github.NewClient(nil)
+	}
+	tokenSource := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: token})
+	return github.NewClient(oauth2.NewClient(ctx, tokenSource))
+}
+
 // NitroReleases returns the most recent 50 releases of the Nitro repository.
 func NitroReleases(ctx context.Context) ([]*github.RepositoryRelease, error) {
-	client := github.NewClient(nil)
+	client := getAuthGitClient(ctx)
 	opts := &github.ListOptions{
 		PerPage: 50,
 	}
@@ -36,7 +47,7 @@ func LatestConsensusRelease(ctx context.Context) (*ConsensusRelease, error) {
 	}
 	var found *ConsensusRelease
 	for _, release := range releases {
-		if strings.HasPrefix(release.GetTagName(), "consensus") {
+		if strings.HasPrefix(release.GetTagName(), "consensus") && !release.GetPrerelease() {
 			if found, err = fromRelease(release); err != nil {
 				return nil, err
 			}