Merge branch 'compatible' into feature/simplify-pickles-types1

.gitignore

@@ -80,6 +80,7 @@ automation/services/watchdog/check_libp2p/check_libp2p
 *.terraform.lock.hcl
 *gcloud-keyfile.json*
 _opam
+opam_switches
 
 target
 ./release
@@ -103,3 +104,7 @@ Pipfile.lock
 # coverage report files
 *.coverage
 _coverage/
+
+# directory removed in this branch, but still existing in berkeley,
+# may appear as an untracked file occasionally
+/src/external

.gitmodules

@@ -1,19 +1,9 @@
 [submodule "src/lib/snarky"]
 	path = src/lib/snarky
 	url = https://github.com/o1-labs/snarky
-[submodule "src/external/ppx_optcomp"]
-	path = src/external/ppx_optcomp
-	url = https://github.com/MinaProtocol/ppx_optcomp
 [submodule "src/lib/crypto/proof-systems"]
 	path = src/lib/crypto/proof-systems
 	url = https://github.com/o1-labs/proof-systems.git
-[submodule "src/external/prometheus"]
-	path = src/external/prometheus
-	url = https://github.com/MinaProtocol/prometheus.git
-    branch = master
-[submodule "src/external/c-reference-signer"]
-	path = src/external/c-reference-signer
-	url = https://github.com/MinaProtocol/c-reference-signer.git
 [submodule "src/lib/crypto/kimchi_bindings/stubs/kimchi-stubs-vendors"]
 	path = src/lib/crypto/kimchi_bindings/stubs/kimchi-stubs-vendors
 	url = https://github.com/MinaProtocol/kimchi-stubs-vendors.git

CODEOWNERS

@@ -11,8 +11,6 @@
 /README.md @MinaProtocol/product-eng-reviewers
 /README-dev.md @MinaProtocol/protocol-eng-reviewers
 
-/src/external/ @bkase @psteckler @mrmr1993
-
 /src/app/archive @MinaProtocol/protocol-eng-reviewers
 /src/app/cli/src/mina.ml @MinaProtocol/protocol-eng-reviewers
 /src/app/cli/src/init @MinaProtocol/protocol-eng-reviewers

Makefile

@@ -12,6 +12,14 @@ ifeq ($(DUNE_PROFILE),)
 DUNE_PROFILE := dev
 endif
 
+ifeq ($(OPAMSWITCH)$(IN_NIX_SHELL)$(CI)$(BUILDKITE),)
+# Sometimes opam replaces these env variables in shell with
+# an explicit mention of a particular switch (dereferenced from the value)
+OPAM_SWITCH_PREFIX := $(PWD)/_opam
+OCAML_TOPLEVEL_PATH := $(OPAM_SWITCH_PREFIX)/lib/toplevel
+PATH := $(OPAM_SWITCH_PREFIX)/bin:$(PATH)
+endif
+
 # Temp directory
 TMPDIR ?= /tmp
 
@@ -43,28 +51,27 @@ clean:
 	@rm -rf src/$(COVERAGE_DIR)
 	@rm -rf src/app/libp2p_helper/result src/libp2p_ipc/libp2p_ipc.capnp.go
 
+switch:
+	./scripts/update-opam-switch.sh
+
 # enforces the OCaml version being used
-ocaml_version:
+ocaml_version: switch
 	@if ! ocamlopt -config | grep "version:" | grep $(OCAML_VERSION); then echo "incorrect OCaml version, expected version $(OCAML_VERSION)" ; exit 1; fi
 
 # enforce machine word size
-ocaml_word_size:
+ocaml_word_size: switch
 	@if ! ocamlopt -config | grep "word_size:" | grep $(WORD_SIZE); then echo "invalid machine word size, expected $(WORD_SIZE)" ; exit 1; fi
 
 
 # Checks that the current opam switch contains the packages from opam.export at the same version.
 # This check is disabled in the pure nix environment (that does not use opam).
-check_opam_switch:
+check_opam_switch: switch
 ifneq ($(DISABLE_CHECK_OPAM_SWITCH), true)
-    ifeq (, $(shell which check_opam_switch))
-	$(warning The check_opam_switch binary was not found in the PATH.)
-	$(error The current opam switch should likely be updated by running: "opam switch import opam.export")
-    else
-	check_opam_switch opam.export
-    endif
+	@which check_opam_switch 2>/dev/null >/dev/null || ( echo "The check_opam_switch binary was not found in the PATH, try: opam switch import opam.export" >&2 && exit 1 )
+	@check_opam_switch opam.export
 endif
 
-ocaml_checks: ocaml_version ocaml_word_size check_opam_switch
+ocaml_checks: switch ocaml_version ocaml_word_size check_opam_switch
 
 libp2p_helper:
 ifeq (, $(MINA_LIBP2P_HELPER_PATH))
@@ -82,22 +89,22 @@ check: ocaml_checks libp2p_helper
 
 build: ocaml_checks reformat-diff libp2p_helper
 	$(info Starting Build)
-	(ulimit -s 65532 || true) && (ulimit -n 10240 || true) && env MINA_COMMIT_SHA1=$(GITLONGHASH) dune build src/app/logproc/logproc.exe src/app/cli/src/mina.exe --profile=$(DUNE_PROFILE)
+	(ulimit -s 65532 || true) && (ulimit -n 10240 || true) && env MINA_COMMIT_SHA1=$(GITLONGHASH) dune build src/app/logproc/logproc.exe src/app/cli/src/mina.exe src/app/generate_keypair/generate_keypair.exe src/app/validate_keypair/validate_keypair.exe src/app/runtime_genesis_ledger/runtime_genesis_ledger.exe --profile=$(DUNE_PROFILE)
 	$(info Build complete)
 
-build_all_sigs: ocaml_checks reformat-diff libp2p_helper
+build_all_sigs: ocaml_checks reformat-diff libp2p_helper build
 	$(info Starting Build)
-	(ulimit -s 65532 || true) && (ulimit -n 10240 || true) && env MINA_COMMIT_SHA1=$(GITLONGHASH) dune build src/app/logproc/logproc.exe src/app/cli/src/mina.exe src/app/cli/src/mina_testnet_signatures.exe src/app/cli/src/mina_mainnet_signatures.exe --profile=$(DUNE_PROFILE)
+	(ulimit -s 65532 || true) && (ulimit -n 10240 || true) && env MINA_COMMIT_SHA1=$(GITLONGHASH) dune build src/app/cli/src/mina_testnet_signatures.exe src/app/cli/src/mina_mainnet_signatures.exe --profile=$(DUNE_PROFILE)
 	$(info Build complete)
 
 build_archive: ocaml_checks reformat-diff
 	$(info Starting Build)
 	(ulimit -s 65532 || true) && (ulimit -n 10240 || true) && dune build src/app/archive/archive.exe --profile=$(DUNE_PROFILE)
 	$(info Build complete)
 
-build_archive_all_sigs: ocaml_checks reformat-diff
+build_archive_utils: ocaml_checks reformat-diff
 	$(info Starting Build)
-	(ulimit -s 65532 || true) && (ulimit -n 10240 || true) && dune build src/app/archive/archive.exe src/app/archive/archive_testnet_signatures.exe src/app/archive/archive_mainnet_signatures.exe --profile=$(DUNE_PROFILE)
+	(ulimit -s 65532 || true) && (ulimit -n 10240 || true) && dune build src/app/archive/archive.exe src/app/replayer/replayer.exe src/app/archive_blocks/archive_blocks.exe src/app/extract_blocks/extract_blocks.exe src/app/missing_blocks_auditor/missing_blocks_auditor.exe --profile=$(DUNE_PROFILE)
 	$(info Build complete)
 
 build_rosetta: ocaml_checks
@@ -201,14 +208,7 @@ publish-macos:
 	@./scripts/publish-macos.sh
 
 deb:
-	./scripts/rebuild-deb.sh
-	./scripts/archive/build-release-archives.sh
-	@mkdir -p /tmp/artifacts
-	@cp _build/mina*.deb /tmp/artifacts/.
-
-deb_optimized:
-	./scripts/rebuild-deb.sh "optimized"
-	./scripts/archive/build-release-archives.sh
+	./scripts/debian/builder.sh
 	@mkdir -p /tmp/artifacts
 	@cp _build/mina*.deb /tmp/artifacts/.
 
@@ -222,14 +222,12 @@ build_or_download_pv_keys: ocaml_checks
 	(ulimit -s 65532 || true) && (ulimit -n 10240 || true) && env MINA_COMMIT_SHA1=$(GITLONGHASH) dune exec --profile=$(DUNE_PROFILE) src/lib/snark_keys/gen_keys/gen_keys.exe -- --generate-keys-only
 	$(info Keys built)
 
-publish_debs:
-	@./buildkite/scripts/publish-deb.sh
-
 genesiskeys:
 	@mkdir -p /tmp/artifacts
 	@cp _build/default/src/lib/key_gen/sample_keypairs.ml /tmp/artifacts/.
 	@cp _build/default/src/lib/key_gen/sample_keypairs.json /tmp/artifacts/.
 
+
 ##############################################
 ## Genesis ledger in OCaml from running daemon
 
@@ -304,4 +302,4 @@ ml-docs: ocaml_checks
 # https://www.gnu.org/software/make/manual/html_node/Phony-Targets.html
 # HACK: cat Makefile | egrep '^\w.*' | sed 's/:/ /' | awk '{print $1}' | grep -v myprocs | sort | xargs
 
-.PHONY: all build check-format clean deb dev mina-docker reformat doc_diagrams ml-docs macos-setup macos-setup-download setup-opam libp2p_helper dhall_types replayer missing_blocks_auditor extract_blocks archive_blocks ocaml_version ocaml_word_size ocaml_checks
+.PHONY: all build check-format clean deb dev mina-docker reformat doc_diagrams ml-docs macos-setup macos-setup-download setup-opam libp2p_helper dhall_types replayer missing_blocks_auditor extract_blocks archive_blocks ocaml_version ocaml_word_size ocaml_checks switch

README-ci-failures.md

@@ -29,8 +29,8 @@ details and links to the failures.
 ## CI environment mismatch
 
 The CI runs its jobs in multiple Docker images. The images that it is using are
-specified in `buildkite/src/Constants/COntainerImages.dhall`: the CI uses all
-Debian images prefixed by `minaToolchainBuster`.
+specified in `buildkite/src/Constants/ContainerImages.dhall`: the CI uses all
+Debian images prefixed by `minaToolchainBullseye`.
 
 These images are generated by the CI itself, in particular based on the content of the
 `dockerfiles` directory and the `opam.export` file (which describes versions of
@@ -46,7 +46,7 @@ In this case, you should:
    [Google Cloud](https://console.cloud.google.com/gcr/images/o1labs-192920/global/mina-toolchain)
    where they have been automatically uploaded. Look for your branch name and
    commit hash in the second column. You should find several images suffixed
-   with `-buster`, `-bullseye`, `-stretch` and `-focal` (Debian versions).
+   with `-bullseye` and `-focal` (Debian versions).
 3. For each such image, retrieve its full name and hash by hovering its link and
    clicking the `Copy full image name` tooltip that appears (or retrieve it on
    the image's page).

README-dev.md

@@ -70,7 +70,8 @@ You can build Mina using Docker. Using Docker works in any dev environment. See
         ```
 
 1. Install [rustup](https://rustup.rs/).
-2. Create your switch with deps `opam switch import --switch mina opam.export`
+2. Add o1-opam-repository with `opam repository add --yes --all --set-default o1-labs https://github.com/o1-labs/opam-repository.git`
+3. Create your switch with deps `opam switch import --switch mina opam.export`
 
     M1- and M2- operating systems experience issues because Homebrew does not link include files automatically.
 
@@ -98,13 +99,13 @@ You can build Mina using Docker. Using Docker works in any dev environment. See
    - Note:If you get conf-openssl install errors, try running `export PKG_CONFIG_PATH=$(brew --prefix [email protected])/lib/pkgconfig` and try `opam switch import opam.export` again.
    - If prompted, run `opam user-setup install` to enable opam-user-setup support for Merlin.
 
-3. Pin dependencies that override opam versions:
+4. Pin dependencies that override opam versions:
 
     ```sh
     scripts/pin-external-packages.sh
     ```
 
-7. Install the correct version of golang:
+5. Install the correct version of golang:
 
    - `goenv init`
    - To make sure the right `goenv` is used, update your shell env script with:
@@ -118,17 +119,17 @@ You can build Mina using Docker. Using Docker works in any dev environment. See
    - `goenv global 1.18.10`
    - Check that the `go version` returns the right version, otherwise you see the message `compile:version "go1.18.10" does not match go tool version "go1.20.2"`. If so, run `brew remove go` or get the matching version.
 
-9.  Invoke `make build`.
+6.  Invoke `make build`.
 
     If you get errors about `libp2p` and `capnp`, try with `brew install capnp`.
 
-9.  For better IDE support, install the OCaml-LSP language server for OCaml:
+7.  For better IDE support, install the OCaml-LSP language server for OCaml:
 
     ```sh
     opam install ocaml-lsp-server
     ```
 
-10. Set up your IDE. See [Customizing your dev environment for autocomplete/merlin](https://github.com/MinaProtocol/mina/blob/develop/README-dev.md#customizing-your-dev-environment-for-autocompletemerlin).
+8. Set up your IDE. See [Customizing your dev environment for autocomplete/merlin](https://github.com/MinaProtocol/mina/blob/develop/README-dev.md#customizing-your-dev-environment-for-autocompletemerlin).
 
 ### Developer Setup (Linux)
 
@@ -139,6 +140,7 @@ Mina has a variety of opam and system dependencies.
 To get all of the required opam dependencies, run:
 
 ```sh
+opam repository add --yes --all --set-default o1-labs https://github.com/o1-labs/opam-repository.git
 opam switch import opam.export
 ```
 
@@ -250,6 +252,47 @@ dune exec src/app/cli/src/mina.exe -- -help
 
 The command line help is the place to learn about other options to the Mina CLI and how to connect to an existing network, such as Mainnet.
 
+## Building debian package locally
+
+Debian package can be build locally by using below commands:
+
+
+1. Build binaries
+```
+make build
+```
+2. Build debian for mina-devet (example for ubuntu 18.04):
+```
+./scripts/debian/build.sh daemon_devnet
+```
+
+## Building docker locally
+
+Prerequisites: 
+
+- debian package previously built
+- aptly app
+
+Steps:
+
+1. Start local debian repository
+```
+./scripts/debian/aptly.sh start -b -c focal -d _build/ -m unstable -l -p 8081
+```
+
+IMPORTANT: debians should be placed in _build folder
+
+2. Build docker:
+```
+./scripts/docker/build.sh --service mina-daemon -v 3.0.0-dkijania-local-debian-build-a099fc7 --network devnet --deb-codename focal --deb-version 3.0.0-dkijania-local-debian-build-a099fc7 
+```
+
+Where:
+
+`-v` - base docker tag
+`--deb-codename` - input debian codename (buster,bullseye etc.)
+`--deb-version` - version of debian which docker will host
+
 ## Using the Makefile
 
 The Makefile contains placeholder targets for all the common tasks that need to be done and automatically knows how to use Docker.

automation/terraform/modules/services/daemon/README.md

@@ -41,17 +41,31 @@ The manual commands to release each container are the following:
 ### Mina-Daemon Container
 
 *(From the root of the `MinaProtocol/mina` repository)*
-`./scripts/release-docker.sh -s mina-daemon -v <major>.<minor>.<patch> --extra-args "--build-arg deb_version=<DEB_VERSION> --build-arg deb_release=<mina package release channel, e.g. alpha>"`
+`./scripts/docker/build.sh -s mina-daemon -v <major>.<minor>.<patch> --deb_version=<DEB_VERSION> --deb_release=<mina package release channel, e.g. alpha>"`
+
+and 
+
+`./scripts/docker/release.sh -s mina-daemon -v <major>.<minor>.<patch> -  --deb_version=<DEB_VERSION> --deb_release=<mina package release channel, e.g. alpha>"`
+
 
 Example:
-`./scripts/release-docker.sh -s mina-daemon -v 0.0.10-beta4 --extra-args "--build-arg deb_version=0.0.10-beta4-fff3b856 --build-arg deb_release=alpha`
+`./scripts/docker/build.sh -s mina-daemon -v 0.0.10-beta4 --deb_version=0.0.10-beta4-fff3b856 --deb_release=alpha`
+
+`./scripts/docker/release.sh -s mina-daemon -v 0.0.10-beta4 --deb_version=0.0.10-beta4-fff3b856 --deb_release=alpha`
 
 The `--extra-args` argument is for passing additional parameters directly to the `docker build` command. It is used here to pass the required Dockerfile variable `'deb_version` but can also be used to override Dockerfile variables with default values like so `--build-arg deb_repo=release`
 
 ### daemon Container
 
-*(From the root of the `MinaProtocol/mina/automation` folder in the mina repository)*
-`./scripts/release-docker.sh -s daemon -v <major>.<minor>.<patch> --extra-args "--build-arg base_image_tag=<docker tag created in first step> "`
+*(From the root of the `MinaProtocol/mina` repository)*
+`./scripts/docker/build.sh -s mina-daemon -v <major>.<minor>.<patch> --deb_version=<DEB_VERSION> --deb_release=<mina package release channel, e.g. alpha>"`
+
+and 
+
+`./scripts/docker/release.sh -s mina-daemon -v <major>.<minor>.<patch> -  --deb_version=<DEB_VERSION> --deb_release=<mina package release channel, e.g. alpha>"`
+
 
 Example:
-`./scripts/release-docker.sh -s daemon -v 0.0.10-beta4 --extra-args "--build-arg base_image_tag=0.0.10-beta4"`
+`./scripts/docker/build.sh -s mina-daemon -v 0.0.10-beta4 --deb_version=0.0.10-beta4-fff3b856 --deb_release=alpha`
+
+`./scripts/docker/release.sh -s mina-daemon -v 0.0.10-beta4 --deb_version=0.0.10-beta4-fff3b856 --deb_release=alpha`

automation/terraform/modules/services/faucet/README.md

@@ -63,14 +63,17 @@ The manual commands to release each container are the following:
 ### Mina-Daemon Container
 
 *(From the root of the `MinaProtocol/mina` repository)*
+`./scripts/docker/build.sh -s mina-daemon -v <major>.<minor>.<patch> --deb_version=<DEB_VERSION> --deb_release=<mina package release channel, e.g. alpha>"`
 
-`./scripts/release-docker.sh -s mina-daemon -v <major>.<minor>.<patch> --extra-args "--build-arg deb_version=<deb_version>"`
+and 
 
-### daemon Container
+`./scripts/docker/release.sh -s mina-daemon -v <major>.<minor>.<patch> -  --deb_version=<DEB_VERSION> --deb_release=<mina package release channel, e.g. alpha>"`
 
-*(From the root of the `MinaProtocol/mina/automation` repository)*
 
-`./scripts/release-docker.sh -s daemon -v <major>.<minor>.<patch> --extra-args "--build-arg base_image_tag=<docker tag created in first step>"`
+Example:
+`./scripts/docker/build.sh -s mina-daemon -v 0.0.10-beta4 --deb_version=0.0.10-beta4-fff3b856 --deb_release=alpha`
+
+`./scripts/docker/release.sh -s mina-daemon -v 0.0.10-beta4 --deb_version=0.0.10-beta4-fff3b856 --deb_release=alpha`
 
 The `--extra-args` argument is for passing additional parameters directly to the `docker build` command. It is used here to pass the required Dockerfile variable `base_image_tag` but can also be used to override Dockerfile variables with default values like so `--build-arg deb_repo=release`
 
@@ -80,5 +83,10 @@ The Faucet Dockerfile lives in the `MinaProtocol/mina` repository [here](https:/
 
 *(From the root of the `MinaProtocol/mina` repository)*
 
-`./scripts/release-docker.sh -s bot -v <major>.<minor>.<patch>`
+```
+./scripts/docker/build.sh -s bot -v <major>.<minor>.<patch>
+./scripts/docker/release.sh -s bot -v <major>.<minor>.<patch>
+```
+
+
 

automation/terraform/modules/services/graphql-proxy/README.md

@@ -54,13 +54,18 @@ The manual commands to release each container are the following:
 
 *(From the root of the `MinaProtocol/mina` repository)*
 
-`./scripts/release-docker.sh -s mina-daemon -v <major>.<minor>.<patch> --extra-args "--build-arg deb_version=<MINA_VERSION>"`
+`./scripts/docker/release.sh -s mina-daemon -v <major>.<minor>.<patch> --deb_version=<MINA_VERSION>"`
+
+and 
+
+`./scripts/docker/release.sh -s mina-daemon -v <major>.<minor>.<patch> -  --deb_version=<DEB_VERSION> --deb_release=<mina package release channel, e.g. alpha>"`
+
 
 ### daemon Container
 
 *(From the root of the `MinaProtocol/mina/automation` repository)*
 
-`./scripts/release-docker.sh -s daemon -v <major>.<minor>.<patch> --extra-args "--build-arg base_image_tag=<docker tag created in first step>"`
+`./scripts/docker/release.sh -s daemon -v <major>.<minor>.<patch> `
 
 The `--extra-args` argument is for passing additional parameters directly to the `docker build` command. It is used here to pass the required Dockerfile variable `base_image_tag` but can also be used to override Dockerfile variables with default values like so `--build-arg deb_repo=release`
 
@@ -70,4 +75,4 @@ The Faucet Dockerfile lives in the `MinaProtocol/mina` repository [here](https:/
 
 *(From the root of the `MinaProtocol/mina` repository)*
 
-`./scripts/release-docker.sh -s graphql-public-proxy -v <major>.<minor>.<patch>`
+`./scripts/docker/release.sh -s graphql-public-proxy -v <major>.<minor>.<patch>`