Remove build deps management

go run supports running an arbitrary go modules like this: go:generate go run golang.org/x/tools/cmd/[email protected] By using that everywhere that our build wanted to demand a previous install of some program, we don't need any of the complexity around `check_deps.sh`, `install_buildtools.sh` and the like. This PR removes them. There shouldn't be a performance problem because the programs will run out of Go's local module cache. Let's see.
algorand · Jan 22, 2024 · 3a7c12f · 3a7c12f
1 parent 877090b
commit 3a7c12f
Show file tree

Hide file tree

Showing 23 changed files with 37 additions and 251 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -597,12 +597,11 @@ commands:
             GOLANG_VERSION=$(./scripts/get_golang_version.sh)
             eval "$(<< parameters.build_dir >>/gimme "${GOLANG_VERSION}")"
             scripts/configure_dev.sh
-            scripts/buildtools/install_buildtools.sh -o "gotest.tools/gotestsum"
             PACKAGES="$(go list ./... | grep -v /go-algorand/test/)"
             export PACKAGE_NAMES=$(echo $PACKAGES | tr -d '\n')
             export PARTITION_TOTAL=${CIRCLE_NODE_TOTAL}
             export PARTITION_ID=${CIRCLE_NODE_INDEX}
-            gotestsum --format standard-verbose --junitfile << parameters.result_path >>/<< parameters.result_subdir >>/${CIRCLE_NODE_INDEX}/results.xml --jsonfile << parameters.result_path >>/<< parameters.result_subdir >>/${CIRCLE_NODE_INDEX}/testresults.json -- --tags "sqlite_unlock_notify sqlite_omit_load_extension" << parameters.short_test_flag >> -race -timeout 1h -coverprofile=coverage.txt -covermode=atomic -p 1 $PACKAGE_NAMES
+            go run gotest.tools/gotestsum@v1.10.0 --format standard-verbose --junitfile << parameters.result_path >>/<< parameters.result_subdir >>/${CIRCLE_NODE_INDEX}/results.xml --jsonfile << parameters.result_path >>/<< parameters.result_subdir >>/${CIRCLE_NODE_INDEX}/testresults.json -- --tags "sqlite_unlock_notify sqlite_omit_load_extension" << parameters.short_test_flag >> -race -timeout 1h -coverprofile=coverage.txt -covermode=atomic -p 1 $PACKAGE_NAMES
       - store_artifacts:
           path: << parameters.result_path >>
           destination: test-results
@@ -663,7 +662,6 @@ commands:
             GOLANG_VERSION=$(./scripts/get_golang_version.sh)
             eval "$(<< parameters.build_dir >>/gimme "${GOLANG_VERSION}")"
             scripts/configure_dev.sh
-            scripts/buildtools/install_buildtools.sh -o "gotest.tools/gotestsum"
             export ALGOTEST=1
             export SHORTTEST=<< parameters.short_test_flag >>
             export TEST_RESULTS=<< parameters.result_path >>/<< parameters.result_subdir >>/${CIRCLE_NODE_INDEX}

diff --git a/Makefile b/Makefile
@@ -35,12 +35,7 @@ GOTAGSLIST          := sqlite_unlock_notify sqlite_omit_load_extension
 # e.g. make GOTAGSCUSTOM=msgtrace
 GOTAGSLIST += ${GOTAGSCUSTOM}
 
-# If available, use gotestsum instead of 'go test'.
-ifeq (, $(shell which gotestsum))
-export GOTESTCOMMAND=go test
-else
-export GOTESTCOMMAND=gotestsum --format pkgname --jsonfile testresults.json --
-endif
+export GOTESTCOMMAND=go run gotest.tools/[email protected]
 
 # M1 Mac--homebrew install location in /opt/homebrew
 ifeq ($(OS_TYPE), darwin)
@@ -99,8 +94,8 @@ fmt:
 fix: build
 	$(GOPATH1)/bin/algofix */
 
-lint: deps
-	$(GOPATH1)/bin/golangci-lint run -c .golangci.yml
+lint:
+	go run github.com/golangci/golangci-lint/cmd/[email protected] run -c .golangci.yml
 
 check_go_version:
 	@if [ $(CURRENT_GO_VERSION_MAJOR) != $(GOLANG_VERSION_BUILD_MAJOR) ]; then \
@@ -127,17 +122,18 @@ cover:
 prof:
 	cd node && go test $(GOTAGS) -cpuprofile=cpu.out -memprofile=mem.out -mutexprofile=mutex.out
 
-generate: deps
+generate:
 	PATH=$(GOPATH1)/bin:$$PATH go generate ./...
 
 msgp: $(patsubst %,%/msgp_gen.go,$(MSGP_GENERATE))
 
-%/msgp_gen.go: deps ALWAYS
+MSGP := go run github.com/algorand/[email protected]
+%/msgp_gen.go: ALWAYS
 		@set +e; \
 		printf "msgp: $(@D)..."; \
-		$(GOPATH1)/bin/msgp -file ./$(@D) -o $@ -warnmask github.com/algorand/go-algorand > ./$@.out 2>&1; \
+		$(MSGP) -file ./$(@D) -o $@ -warnmask github.com/algorand/go-algorand > ./$@.out 2>&1; \
 		if [ "$$?" != "0" ]; then \
-			printf "failed:\n$(GOPATH1)/bin/msgp -file ./$(@D) -o $@ -warnmask github.com/algorand/go-algorand\n"; \
+			printf "failed:\n$(MSGP) -file ./$(@D) -o $@ -warnmask github.com/algorand/go-algorand\n"; \
 			cat ./$@.out; \
 			rm ./$@.out; \
 			exit 1; \
@@ -157,9 +153,6 @@ crypto/libs/$(OS_TYPE)/$(ARCH)/lib/libsodium.a:
 		$(MAKE) && \
 		$(MAKE) install
 
-deps:
-	./scripts/check_deps.sh
-
 # artifacts
 
 # Regenerate kmd swagger spec files

diff --git a/README.md b/README.md
@@ -37,7 +37,6 @@ Initial environment setup:
 git clone https://github.com/algorand/go-algorand
 cd go-algorand
 ./scripts/configure_dev.sh
-./scripts/buildtools/install_buildtools.sh
 ```
 
 At this point, you are ready to build go-algorand. We use `make` and have a

diff --git a/agreement/actions.go b/agreement/actions.go
@@ -26,7 +26,7 @@ import (
 	"github.com/algorand/go-algorand/protocol"
 )
 
-//go:generate stringer -type=actionType
+//go:generate go run golang.org/x/tools/cmd/stringer@v0.17.0 -type=actionType
 type actionType uint8
 
 const (

diff --git a/agreement/coservice.go b/agreement/coservice.go
@@ -22,7 +22,7 @@ import (
 	"github.com/algorand/go-algorand/logging"
 )
 
-//go:generate stringer -type=coserviceType
+//go:generate go run golang.org/x/tools/cmd/stringer@v0.17.0 -type=coserviceType
 const (
 	demuxCoserviceType coserviceType = iota
 	tokenizerCoserviceType

diff --git a/agreement/events.go b/agreement/events.go
@@ -70,7 +70,7 @@ type externalEvent interface {
 // interface.  The semantics of an event depends on the eventType and not on the
 // type of the implementing struct.
 //
-//go:generate stringer -type=eventType
+//go:generate go run golang.org/x/tools/cmd/stringer@v0.17.0 -type=eventType
 type eventType uint8
 
 const (

diff --git a/agreement/router.go b/agreement/router.go
@@ -27,7 +27,7 @@ import (
 //msgp:ignore stateMachineTag
 type stateMachineTag int
 
-//go:generate stringer -type=stateMachineTag
+//go:generate go run golang.org/x/tools/cmd/stringer@v0.17.0 -type=stateMachineTag
 const (
 	demultiplexer stateMachineTag = iota // type demux
 

diff --git a/daemon/algod/api/Makefile b/daemon/algod/api/Makefile
@@ -1,42 +1,37 @@
-GOPATH	:= $(shell go env GOPATH)
-GOPATH1	:= $(firstword $(subst :, ,$(GOPATH)))
-
 # `make all` or just `make` should be appropriate for dev work
 all:	server/v2/generated/model/types.go server/v2/generated/nonparticipating/public/routes.go server/v2/generated/nonparticipating/private/routes.go server/v2/generated/participating/public/routes.go server/v2/generated/participating/private/routes.go server/v2/generated/data/routes.go server/v2/generated/experimental/routes.go
 
 # `make generate` should be able to replace old `generate.sh` script and be appropriate for build system use
-generate:	oapi-codegen all
+generate:	all
 
+CODEGEN := go run github.com/algorand/oapi-codegen/cmd/[email protected]
 server/v2/generated/nonparticipating/public/routes.go:	algod.oas3.yml
-	$(GOPATH1)/bin/oapi-codegen -config ./server/v2/generated/nonparticipating/public/public_routes.yml algod.oas3.yml
+	$(CODEGEN) -config ./server/v2/generated/nonparticipating/public/public_routes.yml algod.oas3.yml
 
 server/v2/generated/nonparticipating/private/routes.go:	algod.oas3.yml
-	$(GOPATH1)/bin/oapi-codegen -config ./server/v2/generated/nonparticipating/private/private_routes.yml algod.oas3.yml
+	$(CODEGEN) -config ./server/v2/generated/nonparticipating/private/private_routes.yml algod.oas3.yml
 
 server/v2/generated/participating/public/routes.go:	algod.oas3.yml
-	$(GOPATH1)/bin/oapi-codegen -config ./server/v2/generated/participating/public/public_routes.yml algod.oas3.yml
+	$(CODEGEN) -config ./server/v2/generated/participating/public/public_routes.yml algod.oas3.yml
 
 server/v2/generated/participating/private/routes.go:	algod.oas3.yml
-	$(GOPATH1)/bin/oapi-codegen -config ./server/v2/generated/participating/private/private_routes.yml algod.oas3.yml
+	$(CODEGEN) -config ./server/v2/generated/participating/private/private_routes.yml algod.oas3.yml
 
 
 server/v2/generated/data/routes.go:	algod.oas3.yml
-	$(GOPATH1)/bin/oapi-codegen -config ./server/v2/generated/data/data_routes.yml algod.oas3.yml
+	$(CODEGEN) -config ./server/v2/generated/data/data_routes.yml algod.oas3.yml
 
 server/v2/generated/experimental/routes.go:	algod.oas3.yml
-	$(GOPATH1)/bin/oapi-codegen -config ./server/v2/generated/experimental/experimental_routes.yml algod.oas3.yml
+	$(CODEGEN) -config ./server/v2/generated/experimental/experimental_routes.yml algod.oas3.yml
 
 server/v2/generated/model/types.go:	algod.oas3.yml
-	$(GOPATH1)/bin/oapi-codegen -config ./server/v2/generated/model/model_types.yml algod.oas3.yml
+	$(CODEGEN) -config ./server/v2/generated/model/model_types.yml algod.oas3.yml
 
 algod.oas3.yml:	algod.oas2.json
 	curl -s -X POST "https://converter.swagger.io/api/convert" -H "accept: application/json" -H "Content-Type: application/json" -d @./algod.oas2.json -o .3tmp.json
 	python3 jsoncanon.py < .3tmp.json > algod.oas3.yml
 	rm -f .3tmp.json
 
-oapi-codegen:	.PHONY
-	../../../scripts/buildtools/install_buildtools.sh -o github.com/algorand/oapi-codegen -c github.com/algorand/oapi-codegen/cmd/oapi-codegen
-
 clean:
 	rm -rf server/v2/generated/model/types.go server/v2/generated/nonparticipating/public/routes.go server/v2/generated/nonparticipating/private/routes.go server/v2/generated/participating/public/routes.go server/v2/generated/participating/private/routes.go server/v2/generated/data/routes.go algod.oas3.yml
 

diff --git a/daemon/kmd/api/api.go b/daemon/kmd/api/api.go
@@ -59,8 +59,8 @@
 // Base path must be a fully specified package name (else, it seems that swagger feeds a relative path to
 // loader.Config.Import(), and that breaks the vendor directory if the source is symlinked from elsewhere)
 //
-//go:generate swagger generate spec -m -o="./swagger.json"
-//go:generate swagger validate ./swagger.json --stop-on-error
+//go:generate go run github.com/go-swagger/go-swagger/cmd/[email protected] generate spec -m -o="./swagger.json"
+//go:generate go run github.com/go-swagger/go-swagger/cmd/[email protected] validate ./swagger.json --stop-on-error
 //go:generate sh ../lib/kmdapi/bundle_swagger_json.sh
 package api
 
@@ -70,7 +70,7 @@ import (
 
 	"github.com/gorilla/mux"
 
-	"github.com/algorand/go-algorand/daemon/kmd/api/v1"
+	v1 "github.com/algorand/go-algorand/daemon/kmd/api/v1"
 	"github.com/algorand/go-algorand/daemon/kmd/lib/kmdapi"
 	"github.com/algorand/go-algorand/daemon/kmd/session"
 	"github.com/algorand/go-algorand/logging"

diff --git a/data/transactions/application.go b/data/transactions/application.go
@@ -58,7 +58,7 @@ const (
 // OnCompletion is an enum representing some layer 1 side effect that an
 // ApplicationCall transaction will have if it is included in a block.
 //
-//go:generate stringer -type=OnCompletion -output=application_string.go
+//go:generate go run golang.org/x/tools/cmd/stringer@v0.17.0 -type=OnCompletion -output=application_string.go
 type OnCompletion uint64
 
 const (

diff --git a/data/transactions/logic/fields.go b/data/transactions/logic/fields.go
@@ -23,7 +23,7 @@ import (
 	"github.com/algorand/go-algorand/protocol"
 )
 
-//go:generate stringer -type=TxnField,GlobalField,AssetParamsField,AppParamsField,AcctParamsField,AssetHoldingField,OnCompletionConstType,EcdsaCurve,EcGroup,Base64Encoding,JSONRefType,VrfStandard,BlockField -output=fields_string.go
+//go:generate go run golang.org/x/tools/cmd/stringer@v0.17.0 -type=TxnField,GlobalField,AssetParamsField,AppParamsField,AcctParamsField,AssetHoldingField,OnCompletionConstType,EcdsaCurve,EcGroup,Base64Encoding,JSONRefType,VrfStandard,BlockField -output=fields_string.go
 
 // FieldSpec unifies the various specs for assembly, disassembly, and doc generation.
 type FieldSpec interface {

diff --git a/docs/messagepack.md b/docs/messagepack.md
@@ -32,7 +32,7 @@ idea of the performance difference, [here are some benchmarks for Transactions](
 
 ## Code generation
 
-Install `msgp` with [install_buildtools.sh](scripts/buildtools/install_buildtools.sh) and use it with `make msgp`.
+Generate messagepack code with `make msgp`.
 
 The generated Marshal and Unmarshal utilities are located in `msgp_gen.go`
 files. Update `MSGP_GENERATE` in the Makefile to add another package.
diff --git a/ledger/store/trackerdb/hashing.go b/ledger/store/trackerdb/hashing.go
@@ -29,7 +29,7 @@ import (
 // By encoding HashKind at a known-offset, it's possible for hash readers to
 // disambiguate the hashed resource.
 //
-//go:generate stringer -type=HashKind
+//go:generate go run golang.org/x/tools/cmd/stringer@v0.17.0 -type=HashKind
 //msgp:ignore HashKind
 type HashKind byte
 

diff --git a/logging/logspec/agreement.go b/logging/logspec/agreement.go
@@ -24,7 +24,7 @@ import (
 // AgreementType is an enum identifying a specific type of AgreementEvent
 // TODO Maybe this should be called AgreementEventType, since these are not actually types of agreements
 //
-//go:generate stringer -type=AgreementType
+//go:generate go run golang.org/x/tools/cmd/stringer@v0.17.0 -type=AgreementType
 type AgreementType int
 
 const (

diff --git a/logging/logspec/ledger.go b/logging/logspec/ledger.go
@@ -24,7 +24,7 @@ import (
 // LedgerType is an enum identifying a specific type of LedgerEvent
 // TODO Maybe this should be called LedgerEventType, since these are not actually types of ledgers
 //
-//go:generate stringer -type=LedgerType
+//go:generate go run golang.org/x/tools/cmd/stringer@v0.17.0 -type=LedgerType
 type LedgerType int
 
 const (

diff --git a/logging/logspec/root.go b/logging/logspec/root.go
@@ -26,7 +26,7 @@ import (
 // TODO Maybe this should be called ComponentEventType (and change Event to ComponentEvent),
 // since these are not actually types of components
 //
-//go:generate stringer -type=Component
+//go:generate go run golang.org/x/tools/cmd/stringer@v0.17.0 -type=Component
 type Component int
 
 const (

diff --git a/scripts/buildtools/install_buildtools.sh b/scripts/buildtools/install_buildtools.sh
diff --git a/scripts/buildtools/versions b/scripts/buildtools/versions