From 17e2ba8c504c090a0cd18d21e829781b40a02c4a Mon Sep 17 00:00:00 2001 From: Nicolas Mahe Date: Sat, 28 Mar 2020 11:45:27 +0700 Subject: [PATCH 01/33] set grace period to 1m to give enough time to the engine to stop running services and stop gracefully --- scripts/dev.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/dev.sh b/scripts/dev.sh index 4dbfd9107..47c158aed 100755 --- a/scripts/dev.sh +++ b/scripts/dev.sh @@ -92,6 +92,7 @@ function start_engine { --publish $MESG_TENDERMINT_RPC_PORT:26657 \ --publish $MESG_PROMETHEUS_RPC_PORT:26660 \ --publish $MESG_COSMOS_RPC_PORT:1317 \ + --stop-grace-period 1m \ mesg/engine:local } From 9121f1ace91da602fac0dc754d905dcff95b7aad Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Sat, 2 May 2020 08:11:35 +0000 Subject: [PATCH 02/33] Bump github.com/sirupsen/logrus from 1.4.2 to 1.5.0 Bumps [github.com/sirupsen/logrus](https://github.com/sirupsen/logrus) from 1.4.2 to 1.5.0. - [Release notes](https://github.com/sirupsen/logrus/releases) - [Changelog](https://github.com/sirupsen/logrus/blob/master/CHANGELOG.md) - [Commits](https://github.com/sirupsen/logrus/compare/v1.4.2...v1.5.0) Signed-off-by: dependabot-preview[bot] --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 71b16e0a3..d23616c29 100644 --- a/go.mod +++ b/go.mod @@ -43,7 +43,7 @@ require ( github.com/prometheus/client_golang v1.5.0 github.com/pseudomuto/protoc-gen-doc v1.3.1 github.com/rcrowley/go-metrics v0.0.0-20190706150252-9beb055b7962 // indirect - github.com/sirupsen/logrus v1.4.2 + github.com/sirupsen/logrus v1.5.0 github.com/spf13/afero v1.2.2 // indirect github.com/spf13/cobra v0.0.6 github.com/spf13/viper v1.6.2 diff --git a/go.sum b/go.sum index fa146c71e..05a9c96d2 100644 --- a/go.sum +++ b/go.sum @@ -484,6 +484,8 @@ github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.5.0 h1:1N5EYkVAPEywqZRJd7cwnRtCb6xJx7NH3T3WUTF980Q= +github.com/sirupsen/logrus v1.5.0/go.mod h1:+F7Ogzej0PZc/94MaYx/nvG9jOFMD2osvC3s+Squfpo= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= From e4981f7ce2e3d0187a39e776e2a3b54bd5f42b2f Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Sat, 2 May 2020 08:11:50 +0000 Subject: [PATCH 03/33] Bump github.com/spf13/viper from 1.6.2 to 1.6.3 Bumps [github.com/spf13/viper](https://github.com/spf13/viper) from 1.6.2 to 1.6.3. - [Release notes](https://github.com/spf13/viper/releases) - [Commits](https://github.com/spf13/viper/compare/v1.6.2...v1.6.3) Signed-off-by: dependabot-preview[bot] --- go.mod | 2 +- go.sum | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 71b16e0a3..4cede217c 100644 --- a/go.mod +++ b/go.mod @@ -46,7 +46,7 @@ require ( github.com/sirupsen/logrus v1.4.2 github.com/spf13/afero v1.2.2 // indirect github.com/spf13/cobra v0.0.6 - github.com/spf13/viper v1.6.2 + github.com/spf13/viper v1.6.3 github.com/stretchr/testify v1.5.1 github.com/tcnksm/ghr v0.13.0 github.com/tendermint/go-amino v0.15.1 diff --git a/go.sum b/go.sum index fa146c71e..e67e7c9be 100644 --- a/go.sum +++ b/go.sum @@ -89,6 +89,7 @@ github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible h1:jFneRYjIvLMLhDLCzuTuU4rSJUjRplcJQ7pD7MnhC04= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= @@ -513,6 +514,8 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= github.com/spf13/viper v1.6.2 h1:7aKfF+e8/k68gda3LOjo5RxiUqddoFxVq4BKBPrxk5E= github.com/spf13/viper v1.6.2/go.mod h1:t3iDnF5Jlj76alVNuyFBk5oUMCvsrkbvZK0WQdfDi5k= +github.com/spf13/viper v1.6.3 h1:pDDu1OyEDTKzpJwdq4TiuLyMsUgRa/BT5cn5O62NoHs= +github.com/spf13/viper v1.6.3/go.mod h1:jUMtyi0/lB5yZH/FjyGAoH7IMNrIhlBf6pXZmbMDvzw= github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= From da8b25ac9a14ebf0b5eac740f6180bce02be91df Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Sat, 2 May 2020 08:12:03 +0000 Subject: [PATCH 04/33] Bump google.golang.org/grpc from 1.28.0 to 1.29.1 Bumps [google.golang.org/grpc](https://github.com/grpc/grpc-go) from 1.28.0 to 1.29.1. - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](https://github.com/grpc/grpc-go/compare/v1.28.0...v1.29.1) Signed-off-by: dependabot-preview[bot] --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 71b16e0a3..f8dd47592 100644 --- a/go.mod +++ b/go.mod @@ -55,7 +55,7 @@ require ( golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413 golang.org/x/net v0.0.0-20200301022130-244492dfa37a // indirect golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527 // indirect - google.golang.org/grpc v1.28.0 + google.golang.org/grpc v1.29.1 gopkg.in/yaml.v2 v2.2.8 gotest.tools v2.2.0+incompatible // indirect ) diff --git a/go.sum b/go.sum index fa146c71e..5cd983299 100644 --- a/go.sum +++ b/go.sum @@ -715,6 +715,8 @@ google.golang.org/grpc v1.27.1 h1:zvIju4sqAGvwKspUQOhwnpcqSbzi7/H6QomNNjTL4sk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.28.0 h1:bO/TA4OxCOummhSf10siHuG7vJOiwh7SpRpFZDkOgl4= google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1 h1:EC2SB8S04d2r73uptxphDSUG+kTKVgjRPF+N3xpxRB4= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From 80d225e08f8e03c2bb7ec5d97758a2ae95ee57d7 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Sat, 2 May 2020 08:33:41 +0000 Subject: [PATCH 05/33] Bump github.com/golang/protobuf from 1.3.4 to 1.4.0 Bumps [github.com/golang/protobuf](https://github.com/golang/protobuf) from 1.3.4 to 1.4.0. - [Release notes](https://github.com/golang/protobuf/releases) - [Commits](https://github.com/golang/protobuf/compare/v1.3.4...v1.4.0) Signed-off-by: dependabot-preview[bot] --- go.mod | 2 +- go.sum | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 38b8ee932..e2096b59d 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,7 @@ require ( github.com/go-playground/validator/v10 v10.2.0 github.com/gogo/protobuf v1.3.1 github.com/golang/mock v1.3.1 // indirect - github.com/golang/protobuf v1.3.4 + github.com/golang/protobuf v1.4.0 github.com/google/uuid v1.1.1 // indirect github.com/gorilla/mux v1.7.3 github.com/grpc-ecosystem/go-grpc-middleware v1.2.0 diff --git a/go.sum b/go.sum index e69ee41f1..53dd4ae75 100644 --- a/go.sum +++ b/go.sum @@ -211,6 +211,12 @@ github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.4 h1:87PNWwrRvUSnqS4dlcBU/ftvOIBep4sYuBLlh6rX2wk= github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0 h1:oOuy+ugB+P/kBdUnG5QaMXSIyJ1q38wWSojYCb3z5VQ= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -722,6 +728,12 @@ google.golang.org/grpc v1.28.0 h1:bO/TA4OxCOummhSf10siHuG7vJOiwh7SpRpFZDkOgl4= google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= google.golang.org/grpc v1.29.1 h1:EC2SB8S04d2r73uptxphDSUG+kTKVgjRPF+N3xpxRB4= google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0 h1:qdOKuR/EIArgaWNjetjgTzgVTAZ+S/WXVrq9HW9zimw= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From 33617e34d8c6bffad622ab79dfc9bf12cfa3e339 Mon Sep 17 00:00:00 2001 From: Nicolas Mahe Date: Sat, 2 May 2020 15:53:50 +0700 Subject: [PATCH 06/33] update protoc to v 3.11.4 --- Dockerfile.tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.tools b/Dockerfile.tools index a31455ec6..34350c959 100644 --- a/Dockerfile.tools +++ b/Dockerfile.tools @@ -9,7 +9,7 @@ RUN apt-get update -y && \ rm -rf /var/lib/apt/lists/* # protobuf(protoc) version. -ARG protobuf=3.9.1 +ARG protobuf=3.11.4 ARG gogoprotobuf=1.3.1 # install protobuf(protoc). From 159e3e5507c99db92350658f7b732ac7a2c958a7 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Sat, 2 May 2020 09:02:59 +0000 Subject: [PATCH 07/33] Bump github.com/prometheus/client_golang from 1.5.0 to 1.6.0 Bumps [github.com/prometheus/client_golang](https://github.com/prometheus/client_golang) from 1.5.0 to 1.6.0. - [Release notes](https://github.com/prometheus/client_golang/releases) - [Changelog](https://github.com/prometheus/client_golang/blob/master/CHANGELOG.md) - [Commits](https://github.com/prometheus/client_golang/compare/v1.5.0...v1.6.0) Signed-off-by: dependabot-preview[bot] --- go.mod | 2 +- go.sum | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index e2096b59d..2543e76a2 100644 --- a/go.mod +++ b/go.mod @@ -40,7 +40,7 @@ require ( github.com/opencontainers/go-digest v1.0.0-rc1 // indirect github.com/opencontainers/image-spec v1.0.1 // indirect github.com/opencontainers/runc v0.1.1 // indirect - github.com/prometheus/client_golang v1.5.0 + github.com/prometheus/client_golang v1.6.0 github.com/pseudomuto/protoc-gen-doc v1.3.1 github.com/rcrowley/go-metrics v0.0.0-20190706150252-9beb055b7962 // indirect github.com/sirupsen/logrus v1.5.0 diff --git a/go.sum b/go.sum index 53dd4ae75..cf36a27f2 100644 --- a/go.sum +++ b/go.sum @@ -446,6 +446,8 @@ github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5Fsn github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= github.com/prometheus/client_golang v1.5.0 h1:Ctq0iGpCmr3jeP77kbF2UxgvRwzWWz+4Bh9/vJTyg1A= github.com/prometheus/client_golang v1.5.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= +github.com/prometheus/client_golang v1.6.0 h1:YVPodQOcK15POxhgARIvnDRVpLcuK8mglnMrWfyrw6A= +github.com/prometheus/client_golang v1.6.0/go.mod h1:ZLOG9ck3JLRdB5MgO8f+lLTe83AXG6ro35rLTxvnIl4= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE= @@ -468,6 +470,7 @@ github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7z github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.8 h1:+fpWZdT24pJBiqJdAwYBjPSk+5YmQzYNPYzQsdzLkt8= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= +github.com/prometheus/procfs v0.0.11/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/pseudomuto/protoc-gen-doc v1.3.1 h1:Segz6bKr2LCo9bZgm+foCYiyfr4s0BurLzH3MDE7wC0= github.com/pseudomuto/protoc-gen-doc v1.3.1/go.mod h1:y5+P6n3iGrbKG+9O04V5ld71in3v/bX88wUwgt+U8EA= @@ -663,12 +666,14 @@ golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42 h1:vEOn+mP2zCOVzKckCZy6YsCtDblrpj/w7B9nxGNELpg= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82 h1:ywK/j/KkyTHcdyYSZNXGjMwgmDSfjglYZ3vStQ/gSCU= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527 h1:uYVVQ9WP/Ds2ROhcaGPeIdVq0RIXVLwsHlnvJ+cT1So= golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= From c7aa7516a873d143ba0097d1a09b985c04174de7 Mon Sep 17 00:00:00 2001 From: Nicolas Mahe Date: Sat, 2 May 2020 16:07:01 +0700 Subject: [PATCH 08/33] run go mod tidy --- go.mod | 1 - go.sum | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 2543e76a2..433c861fd 100644 --- a/go.mod +++ b/go.mod @@ -54,7 +54,6 @@ require ( github.com/tendermint/tm-db v0.5.0 golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413 golang.org/x/net v0.0.0-20200301022130-244492dfa37a // indirect - golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527 // indirect google.golang.org/grpc v1.29.1 gopkg.in/yaml.v2 v2.2.8 gotest.tools v2.2.0+incompatible // indirect diff --git a/go.sum b/go.sum index cf36a27f2..a41c3a965 100644 --- a/go.sum +++ b/go.sum @@ -470,6 +470,7 @@ github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7z github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.8 h1:+fpWZdT24pJBiqJdAwYBjPSk+5YmQzYNPYzQsdzLkt8= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= +github.com/prometheus/procfs v0.0.11 h1:DhHlBtkHWPYi8O2y31JkK0TF+DGM+51OopZjH/Ia5qI= github.com/prometheus/procfs v0.0.11/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/pseudomuto/protoc-gen-doc v1.3.1 h1:Segz6bKr2LCo9bZgm+foCYiyfr4s0BurLzH3MDE7wC0= @@ -671,8 +672,7 @@ golang.org/x/sys v0.0.0-20200116001909-b77594299b42 h1:vEOn+mP2zCOVzKckCZy6YsCtD golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82 h1:ywK/j/KkyTHcdyYSZNXGjMwgmDSfjglYZ3vStQ/gSCU= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527 h1:uYVVQ9WP/Ds2ROhcaGPeIdVq0RIXVLwsHlnvJ+cT1So= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f h1:gWF768j/LaZugp8dyS4UwsslYCYz9XgFxvlgsn0n9H8= golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= From f60d843b95bb56eac4ddc4a17796393112f4a775 Mon Sep 17 00:00:00 2001 From: Nicolas Mahe Date: Fri, 1 May 2020 15:54:04 +0700 Subject: [PATCH 09/33] Add tx commands for modules execution, instance, ownership, process, runner and service --- x/execution/client/cli/tx.go | 73 +++++++++++++++++++++++++++++++++++- x/instance/client/cli/tx.go | 16 +------- x/ownership/client/cli/tx.go | 13 ++++--- x/process/client/cli/tx.go | 63 ++++++++++++++++++++++++++----- x/runner/client/cli/tx.go | 69 +++++++++++++++++++++++++++++----- x/service/client/cli/tx.go | 42 ++++++++++++++++++++- 6 files changed, 232 insertions(+), 44 deletions(-) diff --git a/x/execution/client/cli/tx.go b/x/execution/client/cli/tx.go index acb49edce..ec4fbe4c4 100644 --- a/x/execution/client/cli/tx.go +++ b/x/execution/client/cli/tx.go @@ -1,11 +1,17 @@ package cli import ( + "bufio" "fmt" + "strings" "github.com/cosmos/cosmos-sdk/client" + "github.com/cosmos/cosmos-sdk/client/context" "github.com/cosmos/cosmos-sdk/client/flags" "github.com/cosmos/cosmos-sdk/codec" + sdk "github.com/cosmos/cosmos-sdk/types" + "github.com/cosmos/cosmos-sdk/x/auth" + "github.com/cosmos/cosmos-sdk/x/auth/client/utils" "github.com/mesg-foundation/engine/x/execution/internal/types" "github.com/spf13/cobra" ) @@ -14,12 +20,75 @@ import ( func GetTxCmd(cdc *codec.Codec) *cobra.Command { executionTxCmd := &cobra.Command{ Use: types.ModuleName, - Short: fmt.Sprintf("%s transactions subcommands", types.ModuleName), + Short: fmt.Sprintf("%s transactions subcommands", strings.Title(types.ModuleName)), DisableFlagParsing: true, SuggestionsMinimumDistance: 2, RunE: client.ValidateCmd, } - executionTxCmd.AddCommand(flags.PostCommands()...) + executionTxCmd.AddCommand(flags.PostCommands( + GetCmdCreate(cdc), + GetCmdUpdate(cdc), + )...) return executionTxCmd } + +// GetCmdCreate is the command to create a execution. +func GetCmdCreate(cdc *codec.Codec) *cobra.Command { + return &cobra.Command{ + Use: "create [definition]", + Short: "Creates a new execution", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + inBuf := bufio.NewReader(cmd.InOrStdin()) + cliCtx := context.NewCLIContext().WithCodec(cdc) + + txBldr := auth.NewTxBuilderFromCLI(inBuf).WithTxEncoder(utils.GetTxEncoder(cdc)) + + rawMsg := fmt.Sprintf(`{"type":"execution/create","value":%s}`, args[0]) + var msg types.MsgCreate + if err := cdc.UnmarshalJSON([]byte(rawMsg), &msg); err != nil { + return err + } + if !msg.Signer.Empty() && !msg.Signer.Equals(cliCtx.FromAddress) { + return fmt.Errorf("the signer set in the definition is not the same as the from flag") + } + msg.Signer = cliCtx.FromAddress + if err := msg.ValidateBasic(); err != nil { + return err + } + + return utils.GenerateOrBroadcastMsgs(cliCtx, txBldr, []sdk.Msg{msg}) + }, + } +} + +// GetCmdUpdate is the command to update a execution. +func GetCmdUpdate(cdc *codec.Codec) *cobra.Command { + return &cobra.Command{ + Use: "update [definition]", + Short: "Updates an execution", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + inBuf := bufio.NewReader(cmd.InOrStdin()) + cliCtx := context.NewCLIContext().WithCodec(cdc) + + txBldr := auth.NewTxBuilderFromCLI(inBuf).WithTxEncoder(utils.GetTxEncoder(cdc)) + + rawMsg := fmt.Sprintf(`{"type":"execution/update","value":%s}`, args[0]) + var msg types.MsgUpdate + if err := cdc.UnmarshalJSON([]byte(rawMsg), &msg); err != nil { + return err + } + if !msg.Executor.Empty() && !msg.Executor.Equals(cliCtx.FromAddress) { + return fmt.Errorf("the executor set in the definition is not the same as the from flag") + } + msg.Executor = cliCtx.FromAddress + if err := msg.ValidateBasic(); err != nil { + return err + } + + return utils.GenerateOrBroadcastMsgs(cliCtx, txBldr, []sdk.Msg{msg}) + }, + } +} diff --git a/x/instance/client/cli/tx.go b/x/instance/client/cli/tx.go index 43d50b641..53ccb49bd 100644 --- a/x/instance/client/cli/tx.go +++ b/x/instance/client/cli/tx.go @@ -1,25 +1,11 @@ package cli import ( - "fmt" - - "github.com/cosmos/cosmos-sdk/client" - "github.com/cosmos/cosmos-sdk/client/flags" "github.com/cosmos/cosmos-sdk/codec" - "github.com/mesg-foundation/engine/x/instance/internal/types" "github.com/spf13/cobra" ) // GetTxCmd returns the transaction commands for this module func GetTxCmd(cdc *codec.Codec) *cobra.Command { - ownershipTxCmd := &cobra.Command{ - Use: types.ModuleName, - Short: fmt.Sprintf("%s transactions subcommands", types.ModuleName), - DisableFlagParsing: true, - SuggestionsMinimumDistance: 2, - RunE: client.ValidateCmd, - } - - ownershipTxCmd.AddCommand(flags.PostCommands()...) - return ownershipTxCmd + return nil } diff --git a/x/ownership/client/cli/tx.go b/x/ownership/client/cli/tx.go index d6be73e06..0ba3308cb 100644 --- a/x/ownership/client/cli/tx.go +++ b/x/ownership/client/cli/tx.go @@ -3,6 +3,7 @@ package cli import ( "bufio" "fmt" + "strings" "github.com/cosmos/cosmos-sdk/client" "github.com/cosmos/cosmos-sdk/client/context" @@ -20,7 +21,7 @@ import ( func GetTxCmd(cdc *codec.Codec) *cobra.Command { ownershipTxCmd := &cobra.Command{ Use: types.ModuleName, - Short: fmt.Sprintf("%s transactions subcommands", types.ModuleName), + Short: fmt.Sprintf("%s transactions subcommands", strings.Title(types.ModuleName)), DisableFlagParsing: true, SuggestionsMinimumDistance: 2, RunE: client.ValidateCmd, @@ -35,8 +36,8 @@ func GetTxCmd(cdc *codec.Codec) *cobra.Command { // GetCmdWithdraw is the CLI command for sending a Withdraw transaction func GetCmdWithdraw(cdc *codec.Codec) *cobra.Command { return &cobra.Command{ - Use: "withdraw-coins [resource] [amount]", - Short: "withdraw amount from resource address", + Use: "withdraw-coins [resourceHash] [amount]", + Short: "withdraw coins from a resource", Args: cobra.ExactArgs(2), RunE: func(cmd *cobra.Command, args []string) error { inBuf := bufio.NewReader(cmd.InOrStdin()) @@ -44,17 +45,17 @@ func GetCmdWithdraw(cdc *codec.Codec) *cobra.Command { txBldr := auth.NewTxBuilderFromCLI(inBuf).WithTxEncoder(utils.GetTxEncoder(cdc)) - h, err := hash.Decode(args[0]) + resourceHash, err := hash.Decode(args[0]) if err != nil { return err } msg := types.MsgWithdraw{ Owner: cliCtx.GetFromAddress(), - ResourceHash: h, + ResourceHash: resourceHash, Amount: args[1], } - if err = msg.ValidateBasic(); err != nil { + if err := msg.ValidateBasic(); err != nil { return err } diff --git a/x/process/client/cli/tx.go b/x/process/client/cli/tx.go index 468417750..e439a982d 100644 --- a/x/process/client/cli/tx.go +++ b/x/process/client/cli/tx.go @@ -1,11 +1,18 @@ package cli import ( + "bufio" "fmt" + "strings" "github.com/cosmos/cosmos-sdk/client" + "github.com/cosmos/cosmos-sdk/client/context" "github.com/cosmos/cosmos-sdk/client/flags" "github.com/cosmos/cosmos-sdk/codec" + sdk "github.com/cosmos/cosmos-sdk/types" + "github.com/cosmos/cosmos-sdk/x/auth" + "github.com/cosmos/cosmos-sdk/x/auth/client/utils" + "github.com/mesg-foundation/engine/hash" "github.com/mesg-foundation/engine/x/process/internal/types" "github.com/spf13/cobra" ) @@ -14,7 +21,7 @@ import ( func GetTxCmd(cdc *codec.Codec) *cobra.Command { processTxCmd := &cobra.Command{ Use: types.ModuleName, - Short: fmt.Sprintf("%s transactions subcommands", types.ModuleName), + Short: fmt.Sprintf("%s transactions subcommands", strings.Title(types.ModuleName)), DisableFlagParsing: true, SuggestionsMinimumDistance: 2, RunE: client.ValidateCmd, @@ -28,26 +35,62 @@ func GetTxCmd(cdc *codec.Codec) *cobra.Command { return processTxCmd } +// GetCmdCreate is the command to create a process. func GetCmdCreate(cdc *codec.Codec) *cobra.Command { return &cobra.Command{ - Use: "create [serviceHash] [key=val]...", + Use: "create [definition]", Short: "Creates a new process", - Args: cobra.MinimumNArgs(1), + Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - // TODO: - return nil + inBuf := bufio.NewReader(cmd.InOrStdin()) + cliCtx := context.NewCLIContext().WithCodec(cdc) + + txBldr := auth.NewTxBuilderFromCLI(inBuf).WithTxEncoder(utils.GetTxEncoder(cdc)) + + rawMsg := fmt.Sprintf(`{"type":"process/create","value":%s}`, args[0]) + var msg types.MsgCreate + if err := cdc.UnmarshalJSON([]byte(rawMsg), &msg); err != nil { + return err + } + if !msg.Owner.Empty() && !msg.Owner.Equals(cliCtx.FromAddress) { + return fmt.Errorf("the owner set in the definition is not the same as the from flag") + } + msg.Owner = cliCtx.FromAddress + if err := msg.ValidateBasic(); err != nil { + return err + } + + return utils.GenerateOrBroadcastMsgs(cliCtx, txBldr, []sdk.Msg{msg}) }, } } +// GetCmdDelete is the command to delete a process. func GetCmdDelete(cdc *codec.Codec) *cobra.Command { return &cobra.Command{ - Use: "create [serviceHash] [key=val]...", - Short: "Creates a new process", - Args: cobra.MinimumNArgs(1), + Use: "delete [processHash]", + Short: "Deletes a process", + Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - // TODO: - return nil + inBuf := bufio.NewReader(cmd.InOrStdin()) + cliCtx := context.NewCLIContext().WithCodec(cdc) + + txBldr := auth.NewTxBuilderFromCLI(inBuf).WithTxEncoder(utils.GetTxEncoder(cdc)) + + processHash, err := hash.Decode(args[0]) + if err != nil { + return fmt.Errorf("arg processHash error: %w", err) + } + + msg := types.MsgDelete{ + Hash: processHash, + Owner: cliCtx.FromAddress, + } + if err := msg.ValidateBasic(); err != nil { + return err + } + + return utils.GenerateOrBroadcastMsgs(cliCtx, txBldr, []sdk.Msg{msg}) }, } } diff --git a/x/runner/client/cli/tx.go b/x/runner/client/cli/tx.go index 403e5e54b..0bc1a451c 100644 --- a/x/runner/client/cli/tx.go +++ b/x/runner/client/cli/tx.go @@ -1,11 +1,18 @@ package cli import ( + "bufio" "fmt" + "strings" "github.com/cosmos/cosmos-sdk/client" + "github.com/cosmos/cosmos-sdk/client/context" "github.com/cosmos/cosmos-sdk/client/flags" "github.com/cosmos/cosmos-sdk/codec" + sdk "github.com/cosmos/cosmos-sdk/types" + "github.com/cosmos/cosmos-sdk/x/auth" + "github.com/cosmos/cosmos-sdk/x/auth/client/utils" + "github.com/mesg-foundation/engine/hash" "github.com/mesg-foundation/engine/x/runner/internal/types" "github.com/spf13/cobra" ) @@ -14,7 +21,7 @@ import ( func GetTxCmd(cdc *codec.Codec) *cobra.Command { runnerTxCmd := &cobra.Command{ Use: types.ModuleName, - Short: fmt.Sprintf("%s transactions subcommands", types.ModuleName), + Short: fmt.Sprintf("%s transactions subcommands", strings.Title(types.ModuleName)), DisableFlagParsing: true, SuggestionsMinimumDistance: 2, RunE: client.ValidateCmd, @@ -28,26 +35,70 @@ func GetTxCmd(cdc *codec.Codec) *cobra.Command { return runnerTxCmd } +// GetCmdCreate is the command to create a runner. func GetCmdCreate(cdc *codec.Codec) *cobra.Command { return &cobra.Command{ - Use: "create [runnerHash] [key=val]...", + Use: "create [serviceHash] [envHash]", Short: "Creates a new runner", Args: cobra.MinimumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - // TODO: - return nil + inBuf := bufio.NewReader(cmd.InOrStdin()) + cliCtx := context.NewCLIContext().WithCodec(cdc) + + txBldr := auth.NewTxBuilderFromCLI(inBuf).WithTxEncoder(utils.GetTxEncoder(cdc)) + + serviceHash, err := hash.Decode(args[0]) + if err != nil { + return fmt.Errorf("arg serviceHash error: %w", err) + } + + var envHash hash.Hash + if len(args) >= 2 && args[1] != "" { + if envHash, err = hash.Decode(args[1]); err != nil { + return fmt.Errorf("arg envHash error: %w", err) + } + } + + msg := types.MsgCreate{ + ServiceHash: serviceHash, + EnvHash: envHash, + Owner: cliCtx.FromAddress, + } + if err := msg.ValidateBasic(); err != nil { + return err + } + + return utils.GenerateOrBroadcastMsgs(cliCtx, txBldr, []sdk.Msg{msg}) }, } } +// GetCmdDelete is the command to delete a runner. func GetCmdDelete(cdc *codec.Codec) *cobra.Command { return &cobra.Command{ - Use: "delete [runnerHash] [key=val]...", - Short: "Creates a new runner", - Args: cobra.MinimumNArgs(1), + Use: "delete [runnerHash]", + Short: "Deletes a runner", + Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - // TODO: - return nil + inBuf := bufio.NewReader(cmd.InOrStdin()) + cliCtx := context.NewCLIContext().WithCodec(cdc) + + txBldr := auth.NewTxBuilderFromCLI(inBuf).WithTxEncoder(utils.GetTxEncoder(cdc)) + + runHash, err := hash.Decode(args[0]) + if err != nil { + return fmt.Errorf("arg runHash error: %w", err) + } + + msg := types.MsgDelete{ + Hash: runHash, + Owner: cliCtx.FromAddress, + } + if err := msg.ValidateBasic(); err != nil { + return err + } + + return utils.GenerateOrBroadcastMsgs(cliCtx, txBldr, []sdk.Msg{msg}) }, } } diff --git a/x/service/client/cli/tx.go b/x/service/client/cli/tx.go index dc3ce6ce7..76f05f773 100644 --- a/x/service/client/cli/tx.go +++ b/x/service/client/cli/tx.go @@ -1,11 +1,17 @@ package cli import ( + "bufio" "fmt" + "strings" "github.com/cosmos/cosmos-sdk/client" + "github.com/cosmos/cosmos-sdk/client/context" "github.com/cosmos/cosmos-sdk/client/flags" "github.com/cosmos/cosmos-sdk/codec" + sdk "github.com/cosmos/cosmos-sdk/types" + "github.com/cosmos/cosmos-sdk/x/auth" + "github.com/cosmos/cosmos-sdk/x/auth/client/utils" "github.com/mesg-foundation/engine/x/service/internal/types" "github.com/spf13/cobra" ) @@ -14,13 +20,45 @@ import ( func GetTxCmd(cdc *codec.Codec) *cobra.Command { serviceTxCmd := &cobra.Command{ Use: types.ModuleName, - Short: fmt.Sprintf("%s transactions subcommands", types.ModuleName), + Short: fmt.Sprintf("%s transactions subcommands", strings.Title(types.ModuleName)), DisableFlagParsing: true, SuggestionsMinimumDistance: 2, RunE: client.ValidateCmd, } - serviceTxCmd.AddCommand(flags.PostCommands()...) + serviceTxCmd.AddCommand(flags.PostCommands( + GetCmdCreate(cdc), + )...) return serviceTxCmd } + +// GetCmdCreate is the CLI command for creating a service. +func GetCmdCreate(cdc *codec.Codec) *cobra.Command { + return &cobra.Command{ + Use: "create [definition]", + Short: "create a service from its definition", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + inBuf := bufio.NewReader(cmd.InOrStdin()) + cliCtx := context.NewCLIContext().WithCodec(cdc) + + txBldr := auth.NewTxBuilderFromCLI(inBuf).WithTxEncoder(utils.GetTxEncoder(cdc)) + + rawMsg := fmt.Sprintf(`{"type":"service/create","value":%s}`, args[0]) + var msg types.MsgCreate + if err := cdc.UnmarshalJSON([]byte(rawMsg), &msg); err != nil { + return err + } + if !msg.Owner.Empty() && !msg.Owner.Equals(cliCtx.FromAddress) { + return fmt.Errorf("the owner set in the definition is not the same as the from flag") + } + msg.Owner = cliCtx.FromAddress + if err := msg.ValidateBasic(); err != nil { + return err + } + + return utils.GenerateOrBroadcastMsgs(cliCtx, txBldr, []sdk.Msg{msg}) + }, + } +} From fd297f71b1ddcf5ea96076acfa5e752b31a84396 Mon Sep 17 00:00:00 2001 From: Nicolas Mahe Date: Sun, 3 May 2020 16:49:06 +0700 Subject: [PATCH 10/33] add orchestrator cmd --- cmd/mesg-cli/main.go | 2 + cmd/mesg-cli/orchestrator.go | 132 +++++++++++++++++++++++++++++++++++ 2 files changed, 134 insertions(+) create mode 100644 cmd/mesg-cli/orchestrator.go diff --git a/cmd/mesg-cli/main.go b/cmd/mesg-cli/main.go index 02fe1f197..b9f23e06e 100644 --- a/cmd/mesg-cli/main.go +++ b/cmd/mesg-cli/main.go @@ -52,6 +52,8 @@ func main() { queryCmd(cdc), txCmd(cdc), flags.LineBreak, + orchestratorCmd(cdc), + flags.LineBreak, lcd.ServeCommand(cdc, registerRoutes), flags.LineBreak, keys.Commands(), diff --git a/cmd/mesg-cli/orchestrator.go b/cmd/mesg-cli/orchestrator.go new file mode 100644 index 000000000..18adbcfe3 --- /dev/null +++ b/cmd/mesg-cli/orchestrator.go @@ -0,0 +1,132 @@ +package main + +import ( + "fmt" + "os" + + "github.com/cosmos/cosmos-sdk/client/context" + "github.com/cosmos/cosmos-sdk/client/flags" + "github.com/cosmos/cosmos-sdk/codec" + "github.com/cosmos/cosmos-sdk/crypto/keys" + "github.com/mesg-foundation/engine/cosmos" + "github.com/mesg-foundation/engine/event/publisher" + "github.com/mesg-foundation/engine/ext/xsignal" + "github.com/mesg-foundation/engine/orchestrator" + "github.com/mesg-foundation/engine/server/grpc" + "github.com/spf13/cobra" + "github.com/spf13/viper" + "github.com/tendermint/tendermint/libs/log" +) + +func orchestratorCmd(cdc *codec.Codec) *cobra.Command { + cmd := &cobra.Command{ + Use: "orchestrator", + Short: "Orchestrator subcommands", + } + cmd.AddCommand(flags.GetCommands( + startOrchestratorCmd(cdc), + )...) + return cmd +} + +func startOrchestratorCmd(cdc *codec.Codec) *cobra.Command { + cmd := &cobra.Command{ + Use: "start", + Short: "Start the Orchestrator", + RunE: func(cmd *cobra.Command, args []string) error { + cliCtx := context.NewCLIContext().WithCodec(cdc) + + if viper.GetString(flagMnemonic) == "" { + return fmt.Errorf("flags --mnemonic is required") + } + if cliCtx.ChainID == "" { + return fmt.Errorf("flag --chain-id is required") + } + + logger := log.NewTMLogger(log.NewSyncWriter(os.Stdout)).With("module", "orchestrator") + client, err := cliCtx.GetNode() + if err != nil { + return err + } + + // init rpc client + logger.Info("starting rpc client") + if err := client.Start(); err != nil { + return err + } + defer func() { + logger.Info("stopping rpc client") + if err := client.Stop(); err != nil { + logger.Error(err.Error()) + } + }() + + kb := cosmos.NewInMemoryKeybase() + if _, err := kb.CreateAccount("orchestrator", viper.GetString(flagMnemonic), "", "password", keys.CreateHDPath(0, 0).String(), cosmos.DefaultAlgo); err != nil { + fmt.Println("keybase error") + return err + } + + // create rpc client + rpc, err := cosmos.NewRPC(client, cdc, kb, cliCtx.ChainID, "orchestrator", "password", viper.GetString(flagGasPrices)) + if err != nil { + return err + } + + // init event publisher + ep := publisher.New(rpc) + + // init gRPC server. + logger.Info("starting grpc server") + server := grpc.New(rpc, ep, viper.GetStringSlice(flagAuthorizedPubKeys)) + defer func() { + logger.Info("stopping grpc server") + server.Close() + }() + + go func() { + if err := server.Serve(viper.GetString(flagGrpcAddr)); err != nil { + logger.Error(err.Error()) + panic(err) + } + }() + + // orchestrator + logger.Info("starting orchestrator") + orch := orchestrator.New(rpc, ep, viper.GetString(flagExecPrice)) + defer func() { + logger.Info("stopping orchestrator") + orch.Stop() + }() + go func() { + if err := orch.Start(); err != nil { + logger.Error(err.Error()) + panic(err) + } + }() + go func() { + for err := range orch.ErrC { + logger.Error(err.Error()) + } + }() + + <-xsignal.WaitForInterrupt() + + return nil + }, + } + cmd.Flags().String(flagGrpcAddr, ":50052", "The address for the gRPC server to expose") + cmd.Flags().String(flagAuthorizedPubKeys, "", "The authorized pubkeys to communicate with the gRPC server") + cmd.Flags().String(flagMnemonic, "", "The account's mnemonic that will be used to sign transactions") + cmd.Flags().String(flagGasPrices, "1.0atto", "The gas price to sign tx") + cmd.Flags().String(flagExecPrice, "10000atto", "The execution price to create execution") + return cmd +} + +const ( + flagGrpcAddr = "grpc-addr" + flagAuthorizedPubKeys = "authorized-pubkeys" + flagMnemonic = "mnemonic" + flagGasPrices = "gas-prices" + flagExecPrice = "exec-price" +) From 9680101cc6fad964f7ab0c8776ceb1639ec642bf Mon Sep 17 00:00:00 2001 From: Nicolas Mahe Date: Sun, 3 May 2020 16:51:17 +0700 Subject: [PATCH 11/33] fix xrand init --- ext/xstrings/strings.go | 10 +++++++++- orchestrator/orchestrator.go | 5 +++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/ext/xstrings/strings.go b/ext/xstrings/strings.go index 21025e14b..3bc93fc34 100644 --- a/ext/xstrings/strings.go +++ b/ext/xstrings/strings.go @@ -1,6 +1,14 @@ package xstrings -import "math/rand" +import ( + "math/rand" + + "github.com/mesg-foundation/engine/ext/xrand" +) + +func init() { + xrand.SeedInit() +} // SliceContains returns true if slice a contains e element, false otherwise. func SliceContains(a []string, e string) bool { diff --git a/orchestrator/orchestrator.go b/orchestrator/orchestrator.go index a9f12b0d9..0af227a51 100644 --- a/orchestrator/orchestrator.go +++ b/orchestrator/orchestrator.go @@ -11,6 +11,7 @@ import ( "github.com/mesg-foundation/engine/event" "github.com/mesg-foundation/engine/event/publisher" "github.com/mesg-foundation/engine/execution" + "github.com/mesg-foundation/engine/ext/xrand" "github.com/mesg-foundation/engine/ext/xstrings" "github.com/mesg-foundation/engine/hash" "github.com/mesg-foundation/engine/process" @@ -22,6 +23,10 @@ import ( "github.com/sirupsen/logrus" ) +func init() { + xrand.SeedInit() +} + // New creates a new Process instance func New(rpc *cosmos.RPC, ep *publisher.EventPublisher, execPrice string) *Orchestrator { return &Orchestrator{ From 83adbba896d38a8eb8cd9671374317eb7c4948ac Mon Sep 17 00:00:00 2001 From: Nicolas Mahe Date: Sun, 3 May 2020 16:51:30 +0700 Subject: [PATCH 12/33] add in memory keybase new function to keybase --- cosmos/keybase.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/cosmos/keybase.go b/cosmos/keybase.go index 548195135..42b3b8164 100644 --- a/cosmos/keybase.go +++ b/cosmos/keybase.go @@ -38,6 +38,14 @@ func NewKeybase(dir string) (*Keybase, error) { }, nil } +// NewInMemoryKeybase initializes a in memory keybase. +func NewInMemoryKeybase() *Keybase { + return &Keybase{ + kb: clientkey.NewInMemoryKeyBase(), + privKeysCache: make(map[[sha256.Size]byte]crypto.PrivKey), + } +} + // NewMnemonic returns a new mnemonic phrase. func (kb *Keybase) NewMnemonic() (string, error) { // read entropy seed straight from crypto.Rand and convert to mnemonic From 74dc36c005aea958f880971144360bd255f49086 Mon Sep 17 00:00:00 2001 From: Nicolas Mahe Date: Sun, 3 May 2020 16:53:21 +0700 Subject: [PATCH 13/33] update daemon cli name and default folder --- app/app.go | 4 ++-- cmd/mesg-daemon/main.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/app.go b/app/app.go index 387537f2f..a828290b3 100644 --- a/app/app.go +++ b/app/app.go @@ -40,8 +40,8 @@ var ( // DefaultCLIHome is the default home directories for the application CLI DefaultCLIHome = os.ExpandEnv("$HOME/.mesg-cli") - // DefaultNodeHome sets the folder where the applcation data and configuration will be stored - DefaultNodeHome = os.ExpandEnv("$HOME/.mesg/tendermint") + // DefaultNodeHome sets the folder where the application data and configuration will be stored + DefaultNodeHome = os.ExpandEnv("$HOME/.mesg-node") // ModuleBasics The module BasicManager is in charge of setting up basic, // non-dependant module elements, such as codec registration diff --git a/cmd/mesg-daemon/main.go b/cmd/mesg-daemon/main.go index 1904a3c91..0e332b27e 100644 --- a/cmd/mesg-daemon/main.go +++ b/cmd/mesg-daemon/main.go @@ -60,7 +60,7 @@ func main() { server.AddCommands(ctx, cdc, rootCmd, newApp, exportAppStateAndTMValidators) // prepare and add flags - executor := cli.PrepareBaseCmd(rootCmd, "AU", app.DefaultNodeHome) + executor := cli.PrepareBaseCmd(rootCmd, "MESG", app.DefaultNodeHome) rootCmd.PersistentFlags().UintVar(&invCheckPeriod, flagInvCheckPeriod, 0, "Assert registered invariants every N blocks") if err := executor.Execute(); err != nil { From 045bc702400dc923922b9ee81fd5dd6c9e0a5f2e Mon Sep 17 00:00:00 2001 From: Nicolas Mahe Date: Sun, 3 May 2020 16:55:38 +0700 Subject: [PATCH 14/33] rename in cosmos lcd and rpc clients minGasPrices by simply gasPrices --- cosmos/lcd.go | 34 +++++++++++++++++----------------- cosmos/rpc.go | 32 ++++++++++++++++---------------- 2 files changed, 33 insertions(+), 33 deletions(-) diff --git a/cosmos/lcd.go b/cosmos/lcd.go index b4d31b902..b93e934f7 100644 --- a/cosmos/lcd.go +++ b/cosmos/lcd.go @@ -24,13 +24,13 @@ import ( // LCD is a simple cosmos LCD client. type LCD struct { - endpoint string - cdc *codec.Codec - kb keys.Keybase - chainID string - accName string - accPassword string - minGasPrices sdktypes.DecCoins + endpoint string + cdc *codec.Codec + kb keys.Keybase + chainID string + accName string + accPassword string + gasPrices sdktypes.DecCoins // local state acc *auth.BaseAccount @@ -39,19 +39,19 @@ type LCD struct { } // NewLCD initializes a cosmos LCD client. -func NewLCD(endpoint string, cdc *codec.Codec, kb keys.Keybase, chainID, accName, accPassword, minGasPrices string) (*LCD, error) { - minGasPricesDecoded, err := sdktypes.ParseDecCoins(minGasPrices) +func NewLCD(endpoint string, cdc *codec.Codec, kb keys.Keybase, chainID, accName, accPassword, gasPrices string) (*LCD, error) { + gasPricesDecoded, err := sdktypes.ParseDecCoins(gasPrices) if err != nil { return nil, err } return &LCD{ - endpoint: endpoint, - cdc: cdc, - kb: kb, - chainID: chainID, - accName: accName, - accPassword: accPassword, - minGasPrices: minGasPricesDecoded, + endpoint: endpoint, + cdc: cdc, + kb: kb, + chainID: chainID, + accName: accName, + accPassword: accPassword, + gasPrices: gasPricesDecoded, }, nil } @@ -213,7 +213,7 @@ func (lcd *LCD) createAndSignTx(msgs []sdk.Msg, acc *auth.BaseAccount) (authtype lcd.chainID, "", nil, - lcd.minGasPrices, + lcd.gasPrices, ).WithKeybase(lcd.kb) // calculate gas diff --git a/cosmos/rpc.go b/cosmos/rpc.go index 84c585df7..74b87a99c 100644 --- a/cosmos/rpc.go +++ b/cosmos/rpc.go @@ -24,12 +24,12 @@ import ( // RPC is a tendermint rpc client with helper functions. type RPC struct { rpcclient.Client - cdc *codec.Codec - kb keys.Keybase - chainID string - accName string - accPassword string - minGasPrices sdktypes.DecCoins + cdc *codec.Codec + kb keys.Keybase + chainID string + accName string + accPassword string + gasPrices sdktypes.DecCoins // Local state acc authExported.Account @@ -38,19 +38,19 @@ type RPC struct { } // NewRPC returns a rpc tendermint client. -func NewRPC(client rpcclient.Client, cdc *codec.Codec, kb keys.Keybase, chainID, accName, accPassword, minGasPrices string) (*RPC, error) { - minGasPricesDecoded, err := sdktypes.ParseDecCoins(minGasPrices) +func NewRPC(client rpcclient.Client, cdc *codec.Codec, kb keys.Keybase, chainID, accName, accPassword, gasPrices string) (*RPC, error) { + gasPricesDecoded, err := sdktypes.ParseDecCoins(gasPrices) if err != nil { return nil, err } return &RPC{ - Client: client, - cdc: cdc, - kb: kb, - chainID: chainID, - accName: accName, - accPassword: accPassword, - minGasPrices: minGasPricesDecoded, + Client: client, + cdc: cdc, + kb: kb, + chainID: chainID, + accName: accName, + accPassword: accPassword, + gasPrices: gasPricesDecoded, }, nil } @@ -213,7 +213,7 @@ func (c *RPC) createAndSignTx(msgs []sdktypes.Msg, acc authExported.Account) (te c.chainID, "", nil, - c.minGasPrices, + c.gasPrices, ).WithKeybase(c.kb) // calculate gas From 8068a7b2475a27b0a072d15bd5f21f769bf445a5 Mon Sep 17 00:00:00 2001 From: Nicolas Mahe Date: Sun, 3 May 2020 17:04:48 +0700 Subject: [PATCH 15/33] update readme --- README.md | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index f45da2cec..9974d7561 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # MESG Engine -[Website](https://mesg.com/) - [Docs](https://docs.mesg.com/) - [Forum](https://forum.mesg.com/) - [Chat](https://discordapp.com/invite/SaZ5HcE) - [Blog](https://medium.com/mesg) +[Website](https://mesg.com/) - [Docs](https://docs.mesg.com/) - [Forum](https://forum.mesg.com/) - [Chat](https://discordapp.com/invite/SaZ5HcE) - [Blog](https://blog.mesg.com/) [![GoDoc](https://godoc.org/github.com/mesg-foundation/engine?status.svg)](https://godoc.org/github.com/mesg-foundation/engine) @@ -25,8 +25,6 @@ To help us build and maintain MESG Engine, refer to the [Contribute](#contribute - [Quick Start Guide](#quick-start-guide) - [Services](#services) - [Architecture](#architecture) -- [Marketplace](#marketplace) -- [Roadmap](#roadmap) - [Community](#community) - [Contribute](#contribute) @@ -39,7 +37,7 @@ This step-by-step guide will show you how to create an application that gets the # Services -Services are build and shared on the [Marketplace](https://marketplace.mesg.com/). They are small and reusable pieces of code that, when grouped together, allow developers to build incredible applications with ease. +Services are small and reusable pieces of code that, when grouped together, allow developers to build incredible applications with ease. You can develop a service for absolutely anything you want, as long as it can run inside Docker. Check the [documentation to create your own services](https://docs.mesg.com/guide/service/what-is-a-service.html). @@ -72,15 +70,11 @@ For more info on how to create your events, visit the [Emit an Event](https://do [![MESG Architecture](https://cdn.rawgit.com/mesg-foundation/core/dev/schema1.svg)](https://docs.mesg.com) -# Marketplace - -We have a common place to post all community-developed Services and Applications. Check out the [Marketplace](https://marketplace.mesg.com). - # Community You can find us and other MESG users on the [forum](https://forum.mesg.com). Feel free to check existing posts and help other users of MESG. -Also, be sure to check out the [blog](https://medium.com/mesg) to stay up-to-date with our articles. +Also, be sure to check out the [blog](https://blog.mesg.com/) to stay up-to-date with our articles. # Contribute @@ -88,4 +82,4 @@ Contributions are more than welcome. For more details on how to contribute, plea If you have any questions, please reach out to us directly on [Discord](https://discordapp.com/invite/5tVTHJC). -[![](https://sourcerer.io/fame/NicolasMahe/mesg-foundation/engine/images/0)](https://sourcerer.io/fame/NicolasMahe/mesg-foundation/engine/links/0)[![](https://sourcerer.io/fame/NicolasMahe/mesg-foundation/engine/images/1)](https://sourcerer.io/fame/NicolasMahe/mesg-foundation/engine/links/1)[![](https://sourcerer.io/fame/NicolasMahe/mesg-foundation/engine/images/2)](https://sourcerer.io/fame/NicolasMahe/mesg-foundation/engine/links/2)[![](https://sourcerer.io/fame/NicolasMahe/mesg-foundation/engine/images/3)](https://sourcerer.io/fame/NicolasMahe/mesg-foundation/engine/links/3)[![](https://sourcerer.io/fame/NicolasMahe/mesg-foundation/engine/images/4)](https://sourcerer.io/fame/NicolasMahe/mesg-foundation/engine/links/4)[![](https://sourcerer.io/fame/NicolasMahe/mesg-foundation/engine/images/5)](https://sourcerer.io/fame/NicolasMahe/mesg-foundation/engine/links/5)[![](https://sourcerer.io/fame/NicolasMahe/mesg-foundation/engine/images/6)](https://sourcerer.io/fame/NicolasMahe/mesg-foundation/engine/links/6)[![](https://sourcerer.io/fame/NicolasMahe/mesg-foundation/engine/images/7)](https://sourcerer.io/fame/NicolasMahe/mesg-foundation/engine/links/7) + From 040cf131f6b56796763489605fac8a68d27bc8c2 Mon Sep 17 00:00:00 2001 From: Nicolas Mahe Date: Tue, 5 May 2020 16:38:26 +0700 Subject: [PATCH 16/33] improve error message in orchestrator command --- cmd/mesg-cli/orchestrator.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/mesg-cli/orchestrator.go b/cmd/mesg-cli/orchestrator.go index 18adbcfe3..0f0666671 100644 --- a/cmd/mesg-cli/orchestrator.go +++ b/cmd/mesg-cli/orchestrator.go @@ -37,10 +37,10 @@ func startOrchestratorCmd(cdc *codec.Codec) *cobra.Command { cliCtx := context.NewCLIContext().WithCodec(cdc) if viper.GetString(flagMnemonic) == "" { - return fmt.Errorf("flags --mnemonic is required") + return fmt.Errorf("mnemonic is required. use flag --mnemonic or config file") } if cliCtx.ChainID == "" { - return fmt.Errorf("flag --chain-id is required") + return fmt.Errorf("chain-id is required. use flag --chain-id or config file") } logger := log.NewTMLogger(log.NewSyncWriter(os.Stdout)).With("module", "orchestrator") From bee4221c98f988e33cbc80f59fe34478202d13ba Mon Sep 17 00:00:00 2001 From: Nicolas Mahe Date: Tue, 5 May 2020 16:52:10 +0700 Subject: [PATCH 17/33] add Dockerfile for cli and add command in makefile to build it --- Dockerfile.cli | 27 +++++++++++++++++++++++++++ Dockerfile.tools | 2 +- Makefile | 5 ++++- 3 files changed, 32 insertions(+), 2 deletions(-) create mode 100644 Dockerfile.cli diff --git a/Dockerfile.cli b/Dockerfile.cli new file mode 100644 index 000000000..d9af9d31c --- /dev/null +++ b/Dockerfile.cli @@ -0,0 +1,27 @@ +# base Go image version for building the binaries +FROM golang:1.13.10 AS build +WORKDIR /app + +COPY go.mod go.sum ./ +RUN go mod download + +COPY . . +ARG version + +RUN go build -mod=readonly -o ./bin/mesg-cli -ldflags="-s -w -X 'github.com/cosmos/cosmos-sdk/version.Name=mesg' -X 'github.com/cosmos/cosmos-sdk/version.ServerName=mesg-daemon' -X 'github.com/cosmos/cosmos-sdk/version.ClientName=mesg-cli' -X 'github.com/cosmos/cosmos-sdk/version.Version=$version'" ./cmd/mesg-cli/ +RUN go build -mod=readonly -o ./bin/mesg-daemon -ldflags="-s -w -X 'github.com/cosmos/cosmos-sdk/version.Name=mesg' -X 'github.com/cosmos/cosmos-sdk/version.ServerName=mesg-daemon' -X 'github.com/cosmos/cosmos-sdk/version.ClientName=mesg-cli' -X 'github.com/cosmos/cosmos-sdk/version.Version=$version'" ./cmd/mesg-daemon/ + +# ubuntu image wit binaries for distribution +FROM ubuntu:18.04 +RUN apt-get update && \ + apt-get install -y --no-install-recommends ca-certificates=20180409 && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +WORKDIR /app +ENV PATH="/app:${PATH}" + +COPY --from=build /app/bin/mesg-cli . +COPY --from=build /app/bin/mesg-daemon . + +CMD ["mesg-daemon", "start"] diff --git a/Dockerfile.tools b/Dockerfile.tools index 34350c959..f8ccfb2f6 100644 --- a/Dockerfile.tools +++ b/Dockerfile.tools @@ -1,5 +1,5 @@ # base Go image version. -FROM golang:1.13.0-stretch +FROM golang:1.13.10 WORKDIR /project diff --git a/Makefile b/Makefile index d645e1e60..bf13bef47 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: all build build-cmd-cosmos changelog check-version clean clean-build clean-docker dep dev dev-mon dev-start dev-stop docker-build docker-dev docker-publish docker-publish-dev docker-tools genesis lint protobuf test publish-cmds +.PHONY: all build build-cmd-cosmos changelog check-version clean clean-build clean-docker dep dev dev-mon dev-start dev-stop docker-build docker-dev docker-publish docker-publish-dev docker-tools genesis lint protobuf test publish-cmds build-docker-cli MAJOR_VERSION := $(shell echo $(version) | cut -d . -f 1) MINOR_VERSION := $(shell echo $(version) | cut -d . -f 1-2) @@ -61,6 +61,9 @@ build-cmd: dep go build -mod=readonly -o ./bin/mesg-cli ./cmd/mesg-cli/ go build -mod=readonly -o ./bin/mesg-daemon ./cmd/mesg-daemon/ +build-docker-cli: check-version + docker build -t mesg/engine:cli -f ./Dockerfile.cli --build-arg version=$(version) . + e2e: docker-dev ./scripts/run-e2e.sh From 98cd8987abca9a99c1be0899047f8a7a72488f1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20Mah=C3=A9?= Date: Tue, 5 May 2020 17:01:13 +0700 Subject: [PATCH 18/33] Update Dockerfile.cli --- Dockerfile.cli | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.cli b/Dockerfile.cli index d9af9d31c..9b4d59217 100644 --- a/Dockerfile.cli +++ b/Dockerfile.cli @@ -11,7 +11,7 @@ ARG version RUN go build -mod=readonly -o ./bin/mesg-cli -ldflags="-s -w -X 'github.com/cosmos/cosmos-sdk/version.Name=mesg' -X 'github.com/cosmos/cosmos-sdk/version.ServerName=mesg-daemon' -X 'github.com/cosmos/cosmos-sdk/version.ClientName=mesg-cli' -X 'github.com/cosmos/cosmos-sdk/version.Version=$version'" ./cmd/mesg-cli/ RUN go build -mod=readonly -o ./bin/mesg-daemon -ldflags="-s -w -X 'github.com/cosmos/cosmos-sdk/version.Name=mesg' -X 'github.com/cosmos/cosmos-sdk/version.ServerName=mesg-daemon' -X 'github.com/cosmos/cosmos-sdk/version.ClientName=mesg-cli' -X 'github.com/cosmos/cosmos-sdk/version.Version=$version'" ./cmd/mesg-daemon/ -# ubuntu image wit binaries for distribution +# ubuntu image with binaries for distribution FROM ubuntu:18.04 RUN apt-get update && \ apt-get install -y --no-install-recommends ca-certificates=20180409 && \ From 6f413eb6f77d75f84ba7016adcf2a0ca74d46e89 Mon Sep 17 00:00:00 2001 From: Nicolas Mahe Date: Mon, 4 May 2020 19:20:23 +0700 Subject: [PATCH 19/33] fix runner e2e that block the tests on error --- e2e/runner_test.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/e2e/runner_test.go b/e2e/runner_test.go index f25298e23..f435d6daa 100644 --- a/e2e/runner_test.go +++ b/e2e/runner_test.go @@ -69,9 +69,10 @@ func testRunner(t *testing.T) { require.NoError(t, cont.Start(testServiceStruct, testInstanceHash, testRunnerHash, testInstanceEnvHash, testInstanceEnv, registerPayloadSignature)) }) - // wait for service to be ready - _, err = stream.Recv() - require.NoError(t, err) + t.Run("wait", func(t *testing.T) { + _, err = stream.Recv() + require.NoError(t, err) + }) }) t.Run("get", func(t *testing.T) { From 6ac069ede04b268eaa0a4c0b1f6941384858238f Mon Sep 17 00:00:00 2001 From: Nicolas Mahe Date: Tue, 5 May 2020 17:21:40 +0700 Subject: [PATCH 20/33] generate a dev chain. save every info in dev-chain folder --- dev-chain/README.md | 30 +++ dev-chain/cli/config/config.toml | 6 + dev-chain/validator/config/app.toml | 4 + dev-chain/validator/config/config.toml | 12 ++ dev-chain/validator/config/genesis.json | 196 ++++++++++++++++++ dev-chain/validator/config/node_key.json | 1 + .../validator/config/priv_validator_key.json | 11 + .../validator/data/priv_validator_state.json | 5 + 8 files changed, 265 insertions(+) create mode 100644 dev-chain/README.md create mode 100644 dev-chain/cli/config/config.toml create mode 100644 dev-chain/validator/config/app.toml create mode 100644 dev-chain/validator/config/config.toml create mode 100644 dev-chain/validator/config/genesis.json create mode 100644 dev-chain/validator/config/node_key.json create mode 100644 dev-chain/validator/config/priv_validator_key.json create mode 100644 dev-chain/validator/data/priv_validator_state.json diff --git a/dev-chain/README.md b/dev-chain/README.md new file mode 100644 index 000000000..1d636b10a --- /dev/null +++ b/dev-chain/README.md @@ -0,0 +1,30 @@ +# Dev chain + +For local use only. + +## Genesis file + +See file `dev-genesis.json` + +## Accounts + +### Validator + +Address: `mesg10cjcxje0jjdxzdq5hpqa4dc4znhuxsax2zh7mp` +Pubkey: `mesgpub1addwnpepq0gwhzul5qcycxxs6r7jdzdn5q29mj2a8gmngfrcfa6l8qs2dar4s2jqycv` +Mnemonic: `fruit lock run bike eyebrow unique embrace cost parade welcome more frown oxygen crane club donate grid harsh marriage host skirt sign warfare cup` +Coins: 100000000stake (but used in gentx) + +### CLI + +Address: `mesg1s6mqusxaq93d70jeekqehg7aepwt7zs306ctq7` +Pubkey: `mesgpub1addwnpepqvdrcdvg3x4tf0y5aapn47njxapdu4l0jgsjzcm2klp9a7eztva66eqnadt` +Mnemonic: `spike raccoon obscure program raw large unaware dragon hamster round artist case fall wage sample velvet robust legend identify innocent film coral picture organ` +Coins: 1000000000000000000000000atto and 1000000000stake + +### Orchestrator + +Address: `mesg1t9h20sn3lk2jdnak5eea4lkqxkpwyfaadtqk4t` +Pubkey: `mesgpub1addwnpepqde3ek5edcpxded9w2rw05jm4z3my57f7dey5pwz8hs0uzw0c3xusqmd53d` +Mnemonic: `neutral false together tattoo matrix stamp poem mouse chair chair grain pledge mandate layer shiver embark struggle vicious antenna total faith genre valley mandate` +Coins: 1000000000000000000000000atto diff --git a/dev-chain/cli/config/config.toml b/dev-chain/cli/config/config.toml new file mode 100644 index 000000000..9aeb0e993 --- /dev/null +++ b/dev-chain/cli/config/config.toml @@ -0,0 +1,6 @@ +trust-node = "true" +chain-id = "mesg-dev-chain" + +# orchestrtor +authorized-pubkeys = "mesgpub1addwnpepqvdrcdvg3x4tf0y5aapn47njxapdu4l0jgsjzcm2klp9a7eztva66eqnadt" +mnemonic = "neutral false together tattoo matrix stamp poem mouse chair chair grain pledge mandate layer shiver embark struggle vicious antenna total faith genre valley mandate" diff --git a/dev-chain/validator/config/app.toml b/dev-chain/validator/config/app.toml new file mode 100644 index 000000000..188b3caf1 --- /dev/null +++ b/dev-chain/validator/config/app.toml @@ -0,0 +1,4 @@ +# The minimum gas prices a validator is willing to accept for processing a +# transaction. A transaction's fees must meet the minimum of any denomination +# specified in this config (e.g. 0.25token1;0.0001token2). +minimum-gas-prices = "1.0atto" diff --git a/dev-chain/validator/config/config.toml b/dev-chain/validator/config/config.toml new file mode 100644 index 000000000..7673e561d --- /dev/null +++ b/dev-chain/validator/config/config.toml @@ -0,0 +1,12 @@ +moniker = "developer" +[rpc] +laddr = "tcp://0.0.0.0:26657" +max_subscriptions_per_client = 100 +[consensus] +timeout_commit = "1s" +[p2p] +addr_book_strict = false +[instrumentation] +prometheus = true +[tx_index] +index_all_keys = true diff --git a/dev-chain/validator/config/genesis.json b/dev-chain/validator/config/genesis.json new file mode 100644 index 000000000..b7dee0ba9 --- /dev/null +++ b/dev-chain/validator/config/genesis.json @@ -0,0 +1,196 @@ +{ + "genesis_time": "2020-05-02T09:36:48.27599Z", + "chain_id": "mesg-dev-chain", + "consensus_params": { + "block": { + "max_bytes": "22020096", + "max_gas": "-1", + "time_iota_ms": "1000" + }, + "evidence": { + "max_age_num_blocks": "100000", + "max_age_duration": "172800000000000" + }, + "validator": { + "pub_key_types": [ + "ed25519" + ] + } + }, + "app_hash": "", + "app_state": { + "auth": { + "params": { + "max_memo_characters": "256", + "tx_sig_limit": "7", + "tx_size_cost_per_byte": "10", + "sig_verify_cost_ed25519": "590", + "sig_verify_cost_secp256k1": "1000" + }, + "accounts": [ + { + "type": "cosmos-sdk/Account", + "value": { + "address": "mesg1t9h20sn3lk2jdnak5eea4lkqxkpwyfaadtqk4t", + "coins": [ + { + "denom": "atto", + "amount": "1000000000000000000000000" + } + ], + "public_key": "", + "account_number": 0, + "sequence": 0 + } + }, + { + "type": "cosmos-sdk/Account", + "value": { + "address": "mesg10cjcxje0jjdxzdq5hpqa4dc4znhuxsax2zh7mp", + "coins": [ + { + "denom": "stake", + "amount": "100000000" + } + ], + "public_key": "", + "account_number": 0, + "sequence": 0 + } + }, + { + "type": "cosmos-sdk/Account", + "value": { + "address": "mesg1s6mqusxaq93d70jeekqehg7aepwt7zs306ctq7", + "coins": [ + { + "denom": "atto", + "amount": "1000000000000000000000000" + }, + { + "denom": "stake", + "amount": "1000000000" + } + ], + "public_key": "", + "account_number": 0, + "sequence": 0 + } + } + ] + }, + "params": null, + "service": {}, + "slashing": { + "params": { + "signed_blocks_window": "100", + "min_signed_per_window": "0.500000000000000000", + "downtime_jail_duration": "600000000000", + "slash_fraction_double_sign": "0.050000000000000000", + "slash_fraction_downtime": "0.010000000000000000" + }, + "signing_infos": {}, + "missed_blocks": {} + }, + "instance": {}, + "runner": {}, + "distribution": { + "params": { + "community_tax": "0.020000000000000000", + "base_proposer_reward": "0.010000000000000000", + "bonus_proposer_reward": "0.040000000000000000", + "withdraw_addr_enabled": true + }, + "fee_pool": { + "community_pool": [] + }, + "delegator_withdraw_infos": [], + "previous_proposer": "", + "outstanding_rewards": [], + "validator_accumulated_commissions": [], + "validator_historical_rewards": [], + "validator_current_rewards": [], + "delegator_starting_infos": [], + "validator_slash_events": [] + }, + "supply": { + "supply": [] + }, + "staking": { + "params": { + "unbonding_time": "1814400000000000", + "max_validators": 100, + "max_entries": 7, + "historical_entries": 0, + "bond_denom": "stake" + }, + "last_total_power": "0", + "last_validator_powers": null, + "validators": null, + "delegations": null, + "unbonding_delegations": null, + "redelegations": null, + "exported": false + }, + "bank": { + "send_enabled": true + }, + "process": {}, + "ownership": {}, + "genutil": { + "gentxs": [ + { + "type": "cosmos-sdk/StdTx", + "value": { + "msg": [ + { + "type": "cosmos-sdk/MsgCreateValidator", + "value": { + "description": { + "moniker": "developer", + "identity": "", + "website": "", + "security_contact": "", + "details": "" + }, + "commission": { + "rate": "0.100000000000000000", + "max_rate": "0.200000000000000000", + "max_change_rate": "0.010000000000000000" + }, + "min_self_delegation": "1", + "delegator_address": "mesg10cjcxje0jjdxzdq5hpqa4dc4znhuxsax2zh7mp", + "validator_address": "mesgvaloper10cjcxje0jjdxzdq5hpqa4dc4znhuxsaxkrzt5d", + "pubkey": "mesgvalconspub1zcjduepqsmu3d8sjzfgc9hv994484kh6g7w6k7hwa6pz6ytx8xl02u0vx7uq3c5hd7", + "value": { + "denom": "stake", + "amount": "100000000" + } + } + } + ], + "fee": { + "amount": [], + "gas": "200000" + }, + "signatures": [ + { + "pub_key": { + "type": "tendermint/PubKeySecp256k1", + "value": "A9Dri5+gMEwY0ND9JomzoBRdyV06NzQkeE9184IKb0dY" + }, + "signature": "9BgADIZvkzMA5qPU05MBiF4jpnmFXOvLd2/0U9t4MSsRThtvUYNkKpdRed/I8RVzjC3DTsIpbfo7PA8iWb1Hbg==" + } + ], + "memo": "a4ed9cf59270fd2e867241cd1134bba019e8452f@192.168.1.110:26656" + } + } + ] + }, + "execution": { + "params": { + "minPrice": "10000atto" + } + } + } +} \ No newline at end of file diff --git a/dev-chain/validator/config/node_key.json b/dev-chain/validator/config/node_key.json new file mode 100644 index 000000000..33417858c --- /dev/null +++ b/dev-chain/validator/config/node_key.json @@ -0,0 +1 @@ +{"priv_key":{"type":"tendermint/PrivKeyEd25519","value":"WnB/ngV9sCHm6WruylEat+XRbmhnJ1dgTdAPIIHWiKVeZeftLREe8aUjz7+2QKdKQkJwovw3b0rFa2Z+Z2zLXw=="}} \ No newline at end of file diff --git a/dev-chain/validator/config/priv_validator_key.json b/dev-chain/validator/config/priv_validator_key.json new file mode 100644 index 000000000..d17eae7d0 --- /dev/null +++ b/dev-chain/validator/config/priv_validator_key.json @@ -0,0 +1,11 @@ +{ + "address": "AE7AAC113558ABD2B3D24EDC3774FBE7027F1F35", + "pub_key": { + "type": "tendermint/PubKeyEd25519", + "value": "hvkWnhISUYLdhS1qetr6R52reu7ugi0RZjm+9XHsN7g=" + }, + "priv_key": { + "type": "tendermint/PrivKeyEd25519", + "value": "O8EZc8rbvs5H4+fdFX3uIzxULm4c6/Yxl5+9yIqkc36G+RaeEhJRgt2FLWp62vpHnat67u6CLRFmOb71cew3uA==" + } +} \ No newline at end of file diff --git a/dev-chain/validator/data/priv_validator_state.json b/dev-chain/validator/data/priv_validator_state.json new file mode 100644 index 000000000..ca3ad2f74 --- /dev/null +++ b/dev-chain/validator/data/priv_validator_state.json @@ -0,0 +1,5 @@ +{ + "height": "0", + "round": "0", + "step": 0 +} \ No newline at end of file From 6025ee53b431e3c7159e2b805b9f7897fec03062 Mon Sep 17 00:00:00 2001 From: Nicolas Mahe Date: Tue, 5 May 2020 17:28:59 +0700 Subject: [PATCH 21/33] Add dev cli dockerfile --- .dockerignore | 1 - Dockerfile.cli.dev | 8 ++++++++ Makefile | 5 ++++- scripts/dev-cli.sh | 50 ++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 62 insertions(+), 2 deletions(-) create mode 100644 Dockerfile.cli.dev create mode 100755 scripts/dev-cli.sh diff --git a/.dockerignore b/.dockerignore index 8de3f1617..3110d40ff 100644 --- a/.dockerignore +++ b/.dockerignore @@ -11,7 +11,6 @@ CONTRIBUTING.md schema1.svg .github -/scripts /examples # test & debug files diff --git a/Dockerfile.cli.dev b/Dockerfile.cli.dev new file mode 100644 index 000000000..5be25223c --- /dev/null +++ b/Dockerfile.cli.dev @@ -0,0 +1,8 @@ +ARG from +FROM $from + +COPY ./dev-chain/cli /root/.mesg-cli +COPY ./dev-chain/validator /root/.mesg-node +COPY ./scripts/dev-cli.sh . + +CMD ["bash", "dev-cli.sh"] diff --git a/Makefile b/Makefile index bf13bef47..640212717 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: all build build-cmd-cosmos changelog check-version clean clean-build clean-docker dep dev dev-mon dev-start dev-stop docker-build docker-dev docker-publish docker-publish-dev docker-tools genesis lint protobuf test publish-cmds build-docker-cli +.PHONY: all build build-cmd-cosmos changelog check-version clean clean-build clean-docker dep dev dev-mon dev-start dev-stop docker-build docker-dev docker-publish docker-publish-dev docker-tools genesis lint protobuf test publish-cmds build-docker-cli build-docker-cli-dev MAJOR_VERSION := $(shell echo $(version) | cut -d . -f 1) MINOR_VERSION := $(shell echo $(version) | cut -d . -f 1-2) @@ -64,6 +64,9 @@ build-cmd: dep build-docker-cli: check-version docker build -t mesg/engine:cli -f ./Dockerfile.cli --build-arg version=$(version) . +build-docker-cli-dev: build-docker-cli + docker build -t mesg/engine:cli-dev -f ./Dockerfile.cli.dev --build-arg from=mesg/engine:cli . + e2e: docker-dev ./scripts/run-e2e.sh diff --git a/scripts/dev-cli.sh b/scripts/dev-cli.sh new file mode 100755 index 000000000..f4853e4db --- /dev/null +++ b/scripts/dev-cli.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +# turn on bash's job control +set -m + +# start daemon +mesg-daemon start & +daemon=$! + +# wait 5 sec for the daemon to start rpc server +sleep 5 & +wait $! + +# start lcd +mesg-cli rest-server --laddr tcp://0.0.0.0:1317 & +lcd=$! + +# start orchestrator +mesg-cli orchestrator start & +orchestrator=$! + +# this variable is used to control the monitoring of the child process +monitoring=true + +# function that stop child processes +function stop { + monitoring=false + echo "stopping all child processes" + kill $daemon $lcd $orchestrator + wait $daemon $lcd $orchestrator +} + +# trap both sigint (ctrl+c) and sigterm (OS ask process to be stopped) +trap stop SIGINT +trap stop SIGTERM + +# start the monitoring loop +while $monitoring +do + # check if all sub processes are running correctly + if [[ -n "$(ps -p $daemon -o pid=)" ]] && [[ -n "$(ps -p $lcd -o pid=)" ]] && [[ -n "$(ps -p $orchestrator -o pid=)" ]]; then + sleep 2 & + wait $! + else + # if one child process is not running, stopping all of them and exit with error code 1 + echo "one child process is not running" + stop + exit 1 + fi +done From 9760e7156baa516e9a2c7b61feadc06ccd67c4a7 Mon Sep 17 00:00:00 2001 From: Nicolas Mahe Date: Tue, 5 May 2020 17:36:30 +0700 Subject: [PATCH 22/33] update e2e to work with dev cli image --- .circleci/config.yml | 2 +- Makefile | 2 +- e2e/api_test.go | 9 +++------ e2e/testdata/e2e.config.yml | 8 -------- scripts/run-e2e.sh | 34 ++++++++++++++++++++++------------ 5 files changed, 27 insertions(+), 28 deletions(-) delete mode 100644 e2e/testdata/e2e.config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml index 07497767c..6d113f9d5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -70,7 +70,7 @@ jobs: - run: docker swarm init - <<: *restore_go_cache - <<: *restore_go_path - - run: make e2e + - run: make e2e version=`echo $CIRCLE_SHA1 | cut -c1-7` - <<: *save_go_path - <<: *save_go_cache diff --git a/Makefile b/Makefile index 640212717..eee6cc5b6 100644 --- a/Makefile +++ b/Makefile @@ -67,7 +67,7 @@ build-docker-cli: check-version build-docker-cli-dev: build-docker-cli docker build -t mesg/engine:cli-dev -f ./Dockerfile.cli.dev --build-arg from=mesg/engine:cli . -e2e: docker-dev +e2e: build-docker-cli-dev ./scripts/run-e2e.sh test: dep diff --git a/e2e/api_test.go b/e2e/api_test.go index 3fa52dbde..57905c7a8 100644 --- a/e2e/api_test.go +++ b/e2e/api_test.go @@ -11,7 +11,6 @@ import ( "github.com/cosmos/cosmos-sdk/crypto/keys" sdk "github.com/cosmos/cosmos-sdk/types" - "github.com/cosmos/cosmos-sdk/x/bank" "github.com/mesg-foundation/engine/app" "github.com/mesg-foundation/engine/config" "github.com/mesg-foundation/engine/container" @@ -41,14 +40,14 @@ var ( lcd *cosmos.LCD lcdEngine *cosmos.LCD cliAddress sdk.AccAddress - cliInitialBalance, _ = sdk.ParseCoins("100000000000000000000000000atto") ) const ( lcdEndpoint = "http://127.0.0.1:1317/" pollingInterval = 500 * time.Millisecond // half a block pollingTimeout = 10 * time.Second // 10 blocks - cliAccountMnemonic = "large fork soccer lab answer enlist robust vacant narrow please inmate primary father must add hub shy couch rail video tool marine pill give" + engineMnemonic = "neutral false together tattoo matrix stamp poem mouse chair chair grain pledge mandate layer shiver embark struggle vicious antenna total faith genre valley mandate" + cliAccountMnemonic = "spike raccoon obscure program raw large unaware dragon hamster round artist case fall wage sample velvet robust legend identify innocent film coral picture organ" cliAccountName = "cli" cliAccountPassword = "pass" ) @@ -77,7 +76,7 @@ func TestAPI(t *testing.T) { kb, err = cosmos.NewKeybase(filepath.Join(cfg.Path, cfg.Cosmos.RelativePath)) require.NoError(t, err) // init engine account - engineAcc, err := kb.CreateAccount(cfg.Account.Name, cfg.Account.Mnemonic, "", cfg.Account.Password, keys.CreateHDPath(cfg.Account.Number, cfg.Account.Index).String(), cosmos.DefaultAlgo) + engineAcc, err := kb.CreateAccount(cfg.Account.Name, engineMnemonic, "", cfg.Account.Password, keys.CreateHDPath(cfg.Account.Number, cfg.Account.Index).String(), cosmos.DefaultAlgo) require.NoError(t, err) engineAddress = engineAcc.GetAddress() @@ -89,8 +88,6 @@ func TestAPI(t *testing.T) { // init LCD with engine account and make a transfer to cli account lcdEngine, err = cosmos.NewLCD(lcdEndpoint, cdc, kb, cfg.DevGenesis.ChainID, cfg.Account.Name, cfg.Account.Password, cfg.Cosmos.MinGasPrices) require.NoError(t, err) - _, err = lcdEngine.BroadcastMsg(bank.NewMsgSend(engineAddress, cliAddress, cliInitialBalance)) - require.NoError(t, err) // init container cont, err = container.New(cfg.Name, cfg.Server.Address, cfg.Name) diff --git a/e2e/testdata/e2e.config.yml b/e2e/testdata/e2e.config.yml deleted file mode 100644 index 105e537c7..000000000 --- a/e2e/testdata/e2e.config.yml +++ /dev/null @@ -1,8 +0,0 @@ -authorized_pubkeys: - - mesgpub1addwnpepqg0ujk8vcwq86z8466rznfvfz5rk992wsy0qtczfedgsy3x90mrzkcd5srf -account: - mnemonic: glimpse upon body vast economy give taxi yellow rabbit come click ranch chronic hammer sport near rotate charge lumber chicken cloud base thing forum -tendermint: - config: - consensus: - timeoutcommit: 1s diff --git a/scripts/run-e2e.sh b/scripts/run-e2e.sh index 4b58b3c99..1ad197165 100755 --- a/scripts/run-e2e.sh +++ b/scripts/run-e2e.sh @@ -2,23 +2,33 @@ set -e -export MESG_PATH="$(pwd)"/e2e.test/mesg - -# first run non existing test to detect compilation error quickly +echo "run non existing test to detect compilation error quickly" go test -mod=readonly -v -count=1 ./e2e/... -run=__NONE__ function onexit { - set +e - ./scripts/dev.sh stop - rm -rf "${MESG_PATH}" + docker service rm engine + docker wait $(docker ps -f label=com.docker.swarm.service.name=engine -q) 2> /dev/null + + docker network remove engine } trap onexit EXIT -rm -rf "${MESG_PATH}" -mkdir -p "${MESG_PATH}" -cp "$(pwd)"/e2e/testdata/e2e.config.yml "${MESG_PATH}"/config.yml - -./scripts/dev.sh -q - +if [[ -z $(docker network list -f name="engine" -q) ]]; then + docker network create --driver overlay engine +fi +docker service create \ + --name engine \ + -p 1317:1317 \ + -p 50052:50052 \ + -p 26657:26657 \ + --network engine \ + --label com.docker.stack.namespace=engine \ + mesg/engine:cli-dev + +echo "waiting to give some time to the container to start and run" +sleep 10 & +wait $! + +echo "starting tests" go test -failfast -mod=readonly -v -count=1 ./e2e/... From 562a5e7ae092ac75ce1b477814fbda7ae930a597 Mon Sep 17 00:00:00 2001 From: Nicolas Mahe Date: Tue, 5 May 2020 19:11:08 +0700 Subject: [PATCH 23/33] add suggestion from code review --- cmd/mesg-cli/orchestrator.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/cmd/mesg-cli/orchestrator.go b/cmd/mesg-cli/orchestrator.go index 0f0666671..181de34be 100644 --- a/cmd/mesg-cli/orchestrator.go +++ b/cmd/mesg-cli/orchestrator.go @@ -18,6 +18,11 @@ import ( "github.com/tendermint/tendermint/libs/log" ) +const ( + accName = "orchestrator" + accPass = "password" +) + func orchestratorCmd(cdc *codec.Codec) *cobra.Command { cmd := &cobra.Command{ Use: "orchestrator", @@ -62,13 +67,13 @@ func startOrchestratorCmd(cdc *codec.Codec) *cobra.Command { }() kb := cosmos.NewInMemoryKeybase() - if _, err := kb.CreateAccount("orchestrator", viper.GetString(flagMnemonic), "", "password", keys.CreateHDPath(0, 0).String(), cosmos.DefaultAlgo); err != nil { + if _, err := kb.CreateAccount(accName, viper.GetString(flagMnemonic), "", accPass, keys.CreateHDPath(viper.GetUint32(flagAccNumber), viper.GetUint32(flagAccIndex)).String(), cosmos.DefaultAlgo); err != nil { fmt.Println("keybase error") return err } // create rpc client - rpc, err := cosmos.NewRPC(client, cdc, kb, cliCtx.ChainID, "orchestrator", "password", viper.GetString(flagGasPrices)) + rpc, err := cosmos.NewRPC(client, cdc, kb, cliCtx.ChainID, accName, accPass, viper.GetString(flagGasPrices)) if err != nil { return err } @@ -120,6 +125,8 @@ func startOrchestratorCmd(cdc *codec.Codec) *cobra.Command { cmd.Flags().String(flagMnemonic, "", "The account's mnemonic that will be used to sign transactions") cmd.Flags().String(flagGasPrices, "1.0atto", "The gas price to sign tx") cmd.Flags().String(flagExecPrice, "10000atto", "The execution price to create execution") + cmd.Flags().String(flagAccNumber, "0", "The account number of the hd path to use to derive the mnemonic") + cmd.Flags().String(flagAccIndex, "0", "The account index of the hd path to use to derive the mnemonic") return cmd } @@ -129,4 +136,6 @@ const ( flagMnemonic = "mnemonic" flagGasPrices = "gas-prices" flagExecPrice = "exec-price" + flagAccNumber = "acc-number" + flagAccIndex = "acc-index" ) From f72dc393439b6185be32c6385ee37c50c00a9e97 Mon Sep 17 00:00:00 2001 From: Nicolas Mahe Date: Wed, 6 May 2020 15:07:19 +0700 Subject: [PATCH 24/33] revert golang/protobuf to v1.3.5 and prometheus/client_golang to v1.5.1 --- go.mod | 6 ++++-- go.sum | 18 ++++-------------- 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/go.mod b/go.mod index 433c861fd..09597946c 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,7 @@ require ( github.com/go-playground/validator/v10 v10.2.0 github.com/gogo/protobuf v1.3.1 github.com/golang/mock v1.3.1 // indirect - github.com/golang/protobuf v1.4.0 + github.com/golang/protobuf v1.3.5 github.com/google/uuid v1.1.1 // indirect github.com/gorilla/mux v1.7.3 github.com/grpc-ecosystem/go-grpc-middleware v1.2.0 @@ -40,7 +40,8 @@ require ( github.com/opencontainers/go-digest v1.0.0-rc1 // indirect github.com/opencontainers/image-spec v1.0.1 // indirect github.com/opencontainers/runc v0.1.1 // indirect - github.com/prometheus/client_golang v1.6.0 + github.com/prometheus/client_golang v1.5.1 + github.com/prometheus/procfs v0.0.11 // indirect github.com/pseudomuto/protoc-gen-doc v1.3.1 github.com/rcrowley/go-metrics v0.0.0-20190706150252-9beb055b7962 // indirect github.com/sirupsen/logrus v1.5.0 @@ -54,6 +55,7 @@ require ( github.com/tendermint/tm-db v0.5.0 golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413 golang.org/x/net v0.0.0-20200301022130-244492dfa37a // indirect + golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f // indirect google.golang.org/grpc v1.29.1 gopkg.in/yaml.v2 v2.2.8 gotest.tools v2.2.0+incompatible // indirect diff --git a/go.sum b/go.sum index a41c3a965..3aa1322ea 100644 --- a/go.sum +++ b/go.sum @@ -211,12 +211,8 @@ github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.4 h1:87PNWwrRvUSnqS4dlcBU/ftvOIBep4sYuBLlh6rX2wk= github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= -github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= -github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= -github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0 h1:oOuy+ugB+P/kBdUnG5QaMXSIyJ1q38wWSojYCb3z5VQ= -github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.3.5 h1:F768QJ1E9tib+q5Sc8MkdJi1RxLTbRcTf8LJV56aRls= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -446,8 +442,8 @@ github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5Fsn github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= github.com/prometheus/client_golang v1.5.0 h1:Ctq0iGpCmr3jeP77kbF2UxgvRwzWWz+4Bh9/vJTyg1A= github.com/prometheus/client_golang v1.5.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= -github.com/prometheus/client_golang v1.6.0 h1:YVPodQOcK15POxhgARIvnDRVpLcuK8mglnMrWfyrw6A= -github.com/prometheus/client_golang v1.6.0/go.mod h1:ZLOG9ck3JLRdB5MgO8f+lLTe83AXG6ro35rLTxvnIl4= +github.com/prometheus/client_golang v1.5.1 h1:bdHYieyGlH+6OLEk2YQha8THib30KP0/yD0YH9m6xcA= +github.com/prometheus/client_golang v1.5.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE= @@ -733,12 +729,6 @@ google.golang.org/grpc v1.28.0 h1:bO/TA4OxCOummhSf10siHuG7vJOiwh7SpRpFZDkOgl4= google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= google.golang.org/grpc v1.29.1 h1:EC2SB8S04d2r73uptxphDSUG+kTKVgjRPF+N3xpxRB4= google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= -google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= -google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= -google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= -google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0 h1:qdOKuR/EIArgaWNjetjgTzgVTAZ+S/WXVrq9HW9zimw= -google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From 0f1a6983ed5fc02d3636e5a55900b1c90ae9dc68 Mon Sep 17 00:00:00 2001 From: Nicolas Mahe Date: Wed, 6 May 2020 13:34:51 +0700 Subject: [PATCH 25/33] remove all unused go files --- config/config.go | 190 ------------------- config/config_test.go | 82 -------- core/main.go | 219 ---------------------- cosmos/config.go | 56 +++--- cosmos/genesis.go | 162 ---------------- cosmos/genesis_test.go | 78 -------- cosmos/keybase.go | 3 - cosmos/node.go | 32 ---- e2e/api_test.go | 80 ++++---- e2e/orchestrator_balance_withdraw_test.go | 6 +- logger/init.go | 27 --- logger/tendermint.go | 72 ------- scripts/dev-cli.sh | 50 ----- version/version.go | 4 - 14 files changed, 79 insertions(+), 982 deletions(-) delete mode 100644 config/config.go delete mode 100644 config/config_test.go delete mode 100644 core/main.go delete mode 100644 cosmos/genesis.go delete mode 100644 cosmos/genesis_test.go delete mode 100644 cosmos/node.go delete mode 100644 logger/init.go delete mode 100644 logger/tendermint.go delete mode 100755 scripts/dev-cli.sh delete mode 100644 version/version.go diff --git a/config/config.go b/config/config.go deleted file mode 100644 index 597a2ef8c..000000000 --- a/config/config.go +++ /dev/null @@ -1,190 +0,0 @@ -package config - -import ( - "fmt" - "io/ioutil" - "os" - "path/filepath" - "time" - - "github.com/mesg-foundation/engine/ext/xvalidator" - homedir "github.com/mitchellh/go-homedir" - "github.com/sirupsen/logrus" - tmconfig "github.com/tendermint/tendermint/config" - "gopkg.in/yaml.v2" -) - -const ( - // CosmosBech32MainPrefix defines the main Bech32 prefix. - CosmosBech32MainPrefix = "mesg" - - // CosmosCoinType is the mesg registered coin type from https://github.com/satoshilabs/slips/blob/master/slip-0044.md. - CosmosCoinType = uint32(470) - - // FullFundraiserPath is the parts of the BIP44 HD path that are fixed by what we used during the fundraiser. - FullFundraiserPath = "44'/470'/0'/0/0" -) - -const ( - defaultConfigFileName = "config.yml" - envPathKey = "MESG_PATH" - envNameKey = "MESG_NAME" -) - -// Config contains all the configuration needed. -type Config struct { - Name string `validate:"required" yaml:"-"` - Path string `validate:"required" yaml:"-"` - - DefaultExecutionPrice string `validate:"required"` - - // AuthorizedPubKeys are the bech32 public key of the accounts that are authorized to call the gRPC Admin API. - AuthorizedPubKeys []string `validate:"dive,required,bech32accpubkey" yaml:"authorized_pubkeys"` - - Server struct { - Address string `validate:"required"` - } - - Log struct { - Format string `validate:"required,oneof=json text"` - ForceColors bool - Level string `validate:"required"` - } - - Tendermint struct { - Config *tmconfig.Config `validate:"required"` - RelativePath string `validate:"required"` - } - - Cosmos struct { - RelativePath string `validate:"required"` - - // Minimum gas prices for transactions. - MinGasPrices string `validate:"required,deccoins"` - } - - DevGenesis struct { - ChainID string `validate:"required"` - InitialBalances string `validate:"required,coins"` - } - - Account struct { - Name string `validate:"required"` - Password string `validate:"required"` - Number uint32 - Index uint32 - Mnemonic string `validate:"omitempty,mnemonic"` - } -} - -// defaultConfig creates a new config with default values. -func defaultConfig() (*Config, error) { - home, err := homedir.Dir() - if err != nil { - return nil, err - } - - var c Config - - c.Name = "engine" - c.Path = filepath.Join(home, ".mesg") - - c.DefaultExecutionPrice = "10000atto" // /x/execution/internal/types/params.go#DefaultMinPrice - - c.Server.Address = ":50052" - c.Log.Format = "text" - c.Log.Level = "info" - c.Log.ForceColors = false - - c.Tendermint.RelativePath = "tendermint" - c.Tendermint.Config = tmconfig.DefaultConfig() - c.Tendermint.Config.RPC.ListenAddress = "tcp://0.0.0.0:26657" - c.Tendermint.Config.RPC.MaxSubscriptionsPerClient = 100 - c.Tendermint.Config.P2P.AddrBookStrict = false - c.Tendermint.Config.P2P.AllowDuplicateIP = true - c.Tendermint.Config.Consensus.TimeoutCommit = 5 * time.Second - c.Tendermint.Config.Instrumentation.Prometheus = true - c.Tendermint.Config.Instrumentation.PrometheusListenAddr = "0.0.0.0:26660" - c.Tendermint.Config.TxIndex.IndexAllKeys = true - - c.Cosmos.RelativePath = "cosmos" - c.Cosmos.MinGasPrices = "1.0atto" - - c.DevGenesis.ChainID = "mesg-dev-chain" - c.DevGenesis.InitialBalances = "250000000000000000000000000atto,100000000stake" // 250 000 000 * 10^18 atto + 100,000,000 stake - - c.Account.Name = "engine" - c.Account.Password = "pass" - c.Account.Number = uint32(0) - c.Account.Index = uint32(0) - - return &c, nil -} - -// New returns a Config after loaded ENV and validate the values. -func New() (*Config, error) { - c, err := defaultConfig() - if err != nil { - return nil, err - } - if err := c.load(); err != nil { - return nil, err - } - if err := c.prepare(); err != nil { - return nil, err - } - if err := c.validate(); err != nil { - return nil, err - } - return c, nil -} - -// load reads config from environmental variables. -func (c *Config) load() error { - if envName, ok := os.LookupEnv(envNameKey); ok { - c.Name = envName - } - if envPath, ok := os.LookupEnv(envPathKey); ok { - c.Path = envPath - } - configFilePath := filepath.Join(c.Path, defaultConfigFileName) - if _, err := os.Stat(configFilePath); !os.IsNotExist(err) { - b, err := ioutil.ReadFile(configFilePath) - if err != nil { - return err - } - if err := yaml.UnmarshalStrict(b, c); err != nil { - return err - } - } - c.Tendermint.Config.SetRoot(filepath.Join(c.Path, c.Tendermint.RelativePath)) - return nil -} - -// prepare setups local directories or any other required thing based on config -func (c *Config) prepare() error { - if err := os.MkdirAll(c.Path, os.FileMode(0755)); err != nil { - return err - } - if err := os.MkdirAll(filepath.Dir(c.Tendermint.Config.GenesisFile()), os.FileMode(0755)); err != nil { - return err - } - if err := os.MkdirAll(c.Tendermint.Config.DBDir(), os.FileMode(0755)); err != nil { - return err - } - if err := os.MkdirAll(filepath.Join(c.Path, c.Cosmos.RelativePath), os.FileMode(0755)); err != nil { - return err - } - return nil -} - -// validate checks values and return an error if any validation failed. -func (c *Config) validate() error { - if _, err := logrus.ParseLevel(c.Log.Level); err != nil { - return fmt.Errorf("config log.level error: %w", err) - } - if err := c.Tendermint.Config.ValidateBasic(); err != nil { - return fmt.Errorf("config tendermint error: %w", err) - } - return xvalidator.Struct(c) -} diff --git a/config/config_test.go b/config/config_test.go deleted file mode 100644 index b13480b20..000000000 --- a/config/config_test.go +++ /dev/null @@ -1,82 +0,0 @@ -package config - -import ( - "io/ioutil" - "os" - "path/filepath" - "testing" - "time" - - homedir "github.com/mitchellh/go-homedir" - "github.com/stretchr/testify/require" -) - -func TestNew(t *testing.T) { - _, err := New() - require.NoError(t, err) -} - -func TestDefaultConfig(t *testing.T) { - home, _ := homedir.Dir() - c, err := New() - require.NoError(t, err) - require.Equal(t, ":50052", c.Server.Address) - require.Equal(t, "text", c.Log.Format) - require.Equal(t, "info", c.Log.Level) - require.Equal(t, false, c.Log.ForceColors) - require.Equal(t, filepath.Join(home, ".mesg"), c.Path) - require.Equal(t, "engine", c.Name) - require.Equal(t, "engine", c.Account.Name) - require.Equal(t, "pass", c.Account.Password) - require.Equal(t, uint32(0), c.Account.Number) - require.Equal(t, uint32(0), c.Account.Index) -} - -func TestEnv(t *testing.T) { - os.Setenv(envPathKey, "tempPath") - defer os.Unsetenv(envPathKey) - os.Setenv(envNameKey, "name") - defer os.Unsetenv(envNameKey) - c, err := New() - require.NoError(t, err) - require.Equal(t, "tempPath", c.Path) - require.Equal(t, "name", c.Name) -} - -func TestLoadFromFile(t *testing.T) { - tempPath, _ := ioutil.TempDir("", "TestLoadFromFile") - defer os.RemoveAll(tempPath) - os.Setenv(envPathKey, tempPath) - defer os.Unsetenv(envPathKey) - - t.Run("key does not exist", func(t *testing.T) { - ioutil.WriteFile(filepath.Join(tempPath, defaultConfigFileName), []byte(`foo: bar`), 0644) - _, err := New() - require.Error(t, err) - }) - t.Run("load", func(t *testing.T) { - ioutil.WriteFile(filepath.Join(tempPath, defaultConfigFileName), []byte(`server: - address: :50050 -log: - forcecolors: true -account: - mnemonic: glimpse upon body vast economy give taxi yellow rabbit come click ranch chronic hammer sport near rotate charge lumber chicken cloud base thing forum -cosmos: - mingasprices: 2.0019294mesg -tendermint: - config: - consensus: - timeoutcommit: 1m6s -`), 0644) - - // load - c, err := New() - require.NoError(t, err) - require.Equal(t, ":50050", c.Server.Address) - require.Equal(t, 66*time.Second, c.Tendermint.Config.Consensus.TimeoutCommit) - require.Equal(t, "tcp://0.0.0.0:26657", c.Tendermint.Config.RPC.ListenAddress) - require.Equal(t, tempPath, c.Path) - require.Equal(t, "engine", c.Name) - require.Equal(t, "2.0019294mesg", c.Cosmos.MinGasPrices) - }) -} diff --git a/core/main.go b/core/main.go deleted file mode 100644 index 2e820d143..000000000 --- a/core/main.go +++ /dev/null @@ -1,219 +0,0 @@ -package main - -import ( - "fmt" - "path/filepath" - - bam "github.com/cosmos/cosmos-sdk/baseapp" - cosmosclient "github.com/cosmos/cosmos-sdk/client" - "github.com/cosmos/cosmos-sdk/client/context" - "github.com/cosmos/cosmos-sdk/codec" - "github.com/cosmos/cosmos-sdk/crypto/keys" - authrest "github.com/cosmos/cosmos-sdk/x/auth/client/rest" - "github.com/gorilla/mux" - "github.com/mesg-foundation/engine/app" - "github.com/mesg-foundation/engine/config" - "github.com/mesg-foundation/engine/cosmos" - "github.com/mesg-foundation/engine/event/publisher" - "github.com/mesg-foundation/engine/ext/xrand" - "github.com/mesg-foundation/engine/ext/xsignal" - "github.com/mesg-foundation/engine/logger" - "github.com/mesg-foundation/engine/orchestrator" - "github.com/mesg-foundation/engine/server/grpc" - "github.com/mesg-foundation/engine/version" - "github.com/sirupsen/logrus" - rpcclient "github.com/tendermint/tendermint/rpc/client" - rpcserver "github.com/tendermint/tendermint/rpc/lib/server" - tmtypes "github.com/tendermint/tendermint/types" - db "github.com/tendermint/tm-db" -) - -func loadOrGenConfigAccount(kb *cosmos.Keybase, cfg *config.Config) (keys.Info, error) { - if cfg.Account.Mnemonic != "" { - logrus.WithField("module", "main").Warn("Config account mnemonic presents. Generating account with it...") - return kb.CreateAccount(cfg.Account.Name, cfg.Account.Mnemonic, "", cfg.Account.Password, keys.CreateHDPath(cfg.Account.Number, cfg.Account.Index).String(), cosmos.DefaultAlgo) - } - - exist, err := kb.Exist(cfg.Account.Name) - if err != nil { - return nil, err - } - if exist { - return kb.Get(cfg.Account.Name) - } - logrus.WithField("module", "main").Warn("Config account not found. Generating one for development...") - mnemonic, err := kb.NewMnemonic() - if err != nil { - return nil, err - } - logrus.WithField("module", "main").WithFields(map[string]interface{}{ - "name": cfg.Account.Name, - "password": cfg.Account.Password, - "mnemonic": mnemonic, - }).Warn("Account") - return kb.CreateAccount(cfg.Account.Name, mnemonic, "", cfg.Account.Password, keys.CreateHDPath(cfg.Account.Number, cfg.Account.Index).String(), cosmos.DefaultAlgo) -} - -func loadOrGenDevGenesis(cdc *codec.Codec, kb *cosmos.Keybase, cfg *config.Config) (*tmtypes.GenesisDoc, error) { - if cosmos.GenesisExist(cfg.Tendermint.Config.GenesisFile()) { - return cosmos.LoadGenesis(cfg.Tendermint.Config.GenesisFile()) - } - // generate dev genesis - logrus.WithField("module", "main").Warn("Genesis file not found. Generating one for development...") - validator, err := cosmos.NewGenesisValidator(kb, - cfg.Account.Name, - cfg.Account.Password, - cfg.Tendermint.Config.PrivValidatorKeyFile(), - cfg.Tendermint.Config.PrivValidatorStateFile(), - cfg.Tendermint.Config.NodeKeyFile(), - ) - if err != nil { - return nil, err - } - logrus.WithField("module", "main").WithFields(map[string]interface{}{ - "nodeID": validator.NodeID, - "peer": fmt.Sprintf("%s@%s:26656", validator.NodeID, validator.Name), - }).Warnln("Validator") - return cosmos.GenGenesis(cdc, kb, app.NewDefaultGenesisState(), cfg.DevGenesis.ChainID, cfg.DevGenesis.InitialBalances, cfg.Tendermint.Config.GenesisFile(), []cosmos.GenesisValidator{validator}) -} - -//nolint:gocyclo -func main() { - xrand.SeedInit() - - // init the config of cosmos - cosmos.InitConfig() - - // load engine config - cfg, err := config.New() - if err != nil { - logrus.WithField("module", "main").Fatalln(err) - } - - // init logger. - logger.Init(cfg.Log.Format, cfg.Log.Level, cfg.Log.ForceColors) - - // init tendermint logger - tendermintLogger := logger.TendermintLogger() - - // init app factory - db, err := db.NewGoLevelDB("app", filepath.Join(cfg.Path, cfg.Cosmos.RelativePath)) - if err != nil { - logrus.WithField("module", "main").Fatalln(err) - } - - initApp, err := app.NewInitApp(tendermintLogger, db, nil, true, 0, bam.SetMinGasPrices(cfg.Cosmos.MinGasPrices)) - if err != nil { - logrus.WithField("module", "main").Fatalln(err) - } - cdc := initApp.Codec() - - // init key manager - kb, err := cosmos.NewKeybase(filepath.Join(cfg.Path, cfg.Cosmos.RelativePath)) - if err != nil { - logrus.WithField("module", "main").Fatalln(err) - } - - // gen config account - acc, err := loadOrGenConfigAccount(kb, cfg) - if err != nil { - logrus.WithField("module", "main").Fatalln(err) - } - logrus.WithField("address", acc.GetAddress().String()).Info("engine account") - - // load or gen genesis - genesis, err := loadOrGenDevGenesis(cdc, kb, cfg) - if err != nil { - logrus.WithField("module", "main").Fatalln(err) - } - - // create cosmos node - node, err := cosmos.NewNode(initApp.BaseApp, cfg.Tendermint.Config, genesis) - if err != nil { - logrus.WithField("module", "main").Fatalln(err) - } - defer func() { - logrus.WithField("module", "main").Info("stopping tendermint") - if node.IsRunning() { - if err := node.Stop(); err != nil { - logrus.WithField("module", "main").Errorln(err) - } - } - }() - - // create rpc client - rpc, err := cosmos.NewRPC(rpcclient.NewLocal(node), cdc, kb, genesis.ChainID, cfg.Account.Name, cfg.Account.Password, cfg.Cosmos.MinGasPrices) - if err != nil { - logrus.WithField("module", "main").Fatalln(err) - } - - // init event publisher - ep := publisher.New(rpc) - - // start tendermint node - logrus.WithField("module", "main").WithField("seeds", cfg.Tendermint.Config.P2P.Seeds).Info("starting tendermint node") - if err := node.Start(); err != nil { - logrus.WithField("module", "main").Fatalln(err) - } - - // init gRPC server. - server := grpc.New(rpc, ep, cfg.AuthorizedPubKeys) - logrus.WithField("module", "main").Infof("starting MESG Engine version %s", version.Version) - defer func() { - logrus.WithField("module", "main").Info("stopping grpc server") - server.Close() - }() - - go func() { - if err := server.Serve(cfg.Server.Address); err != nil { - logrus.WithField("module", "main").Fatalln(err) - } - }() - - logrus.WithField("module", "main").Info("starting process engine") - orch := orchestrator.New(rpc, ep, cfg.DefaultExecutionPrice) - defer func() { - logrus.WithField("module", "main").Info("stopping orchestrator") - orch.Stop() - }() - go func() { - if err := orch.Start(); err != nil { - logrus.WithField("module", "main").Fatalln(err) - } - }() - go func() { - for err := range orch.ErrC { - logrus.WithField("module", "orchestrator").Warn(err) - } - }() - - logrus.WithField("module", "main").Info("starting lcd server") - cfgLcd := rpcserver.DefaultConfig() - lcdServer, err := rpcserver.Listen("tcp://[::]:1317", cfgLcd) - if err != nil { - logrus.WithField("module", "main").Fatalln(err) - } - defer func() { - logrus.WithField("module", "main").Info("stopping lcd server") - if err := lcdServer.Close(); err != nil { - logrus.WithField("module", "main").Errorln(err) - } - }() - - cliCtx := context.NewCLIContext(). - WithCodec(cdc). - WithClient(rpc). - WithTrustNode(true) - mux := mux.NewRouter() - cosmosclient.RegisterRoutes(cliCtx, mux) - authrest.RegisterTxRoutes(cliCtx, mux) - app.ModuleBasics.RegisterRESTRoutes(cliCtx, mux) - cosmos.RegisterSimulateRoute(cliCtx, mux) - go func() { - if err := rpcserver.StartHTTPServer(lcdServer, mux, tendermintLogger, cfgLcd); err != nil { - logrus.WithField("module", "main").Warnln(err) // not a fatal because closing the connection return an error here - } - }() - - <-xsignal.WaitForInterrupt() -} diff --git a/cosmos/config.go b/cosmos/config.go index 110929904..c03f5e6a0 100644 --- a/cosmos/config.go +++ b/cosmos/config.go @@ -1,36 +1,50 @@ package cosmos import ( + "github.com/cosmos/cosmos-sdk/crypto/keys" sdktypes "github.com/cosmos/cosmos-sdk/types" - "github.com/mesg-foundation/engine/config" +) + +const ( + // CosmosBech32MainPrefix defines the main Bech32 prefix. + CosmosBech32MainPrefix = "mesg" + + // CosmosCoinType is the mesg registered coin type from https://github.com/satoshilabs/slips/blob/master/slip-0044.md. + CosmosCoinType = uint32(470) + + // FullFundraiserPath is the parts of the BIP44 HD path that are fixed by what we used during the fundraiser. + FullFundraiserPath = "44'/470'/0'/0/0" + + // GasAdjustment is a multiplier to make sure transactions have enough gas when gas are estimated. + GasAdjustment = 1.5 + + // DefaultAlgo for create account. + DefaultAlgo = keys.Secp256k1 ) // InitConfig sets the bech32 prefix and HDPath to cosmos config. func InitConfig() { // See github.com/cosmos/cosmos-sdk/types/address.go const ( - // Bech32PrefixAccAddr defines the Bech32 prefix of an account's address - Bech32PrefixAccAddr = config.CosmosBech32MainPrefix - // Bech32PrefixAccPub defines the Bech32 prefix of an account's public key - Bech32PrefixAccPub = config.CosmosBech32MainPrefix + sdktypes.PrefixPublic - // Bech32PrefixValAddr defines the Bech32 prefix of a validator's operator address - Bech32PrefixValAddr = config.CosmosBech32MainPrefix + sdktypes.PrefixValidator + sdktypes.PrefixOperator - // Bech32PrefixValPub defines the Bech32 prefix of a validator's operator public key - Bech32PrefixValPub = config.CosmosBech32MainPrefix + sdktypes.PrefixValidator + sdktypes.PrefixOperator + sdktypes.PrefixPublic - // Bech32PrefixConsAddr defines the Bech32 prefix of a consensus node address - Bech32PrefixConsAddr = config.CosmosBech32MainPrefix + sdktypes.PrefixValidator + sdktypes.PrefixConsensus - // Bech32PrefixConsPub defines the Bech32 prefix of a consensus node public key - Bech32PrefixConsPub = config.CosmosBech32MainPrefix + sdktypes.PrefixValidator + sdktypes.PrefixConsensus + sdktypes.PrefixPublic + // bech32PrefixAccAddr defines the Bech32 prefix of an account's address + bech32PrefixAccAddr = CosmosBech32MainPrefix + // bech32PrefixAccPub defines the Bech32 prefix of an account's public key + bech32PrefixAccPub = CosmosBech32MainPrefix + sdktypes.PrefixPublic + // bech32PrefixValAddr defines the Bech32 prefix of a validator's operator address + bech32PrefixValAddr = CosmosBech32MainPrefix + sdktypes.PrefixValidator + sdktypes.PrefixOperator + // bech32PrefixValPub defines the Bech32 prefix of a validator's operator public key + bech32PrefixValPub = CosmosBech32MainPrefix + sdktypes.PrefixValidator + sdktypes.PrefixOperator + sdktypes.PrefixPublic + // bech32PrefixConsAddr defines the Bech32 prefix of a consensus node address + bech32PrefixConsAddr = CosmosBech32MainPrefix + sdktypes.PrefixValidator + sdktypes.PrefixConsensus + // bech32PrefixConsPub defines the Bech32 prefix of a consensus node public key + bech32PrefixConsPub = CosmosBech32MainPrefix + sdktypes.PrefixValidator + sdktypes.PrefixConsensus + sdktypes.PrefixPublic ) cfg := sdktypes.GetConfig() - cfg.SetBech32PrefixForAccount(Bech32PrefixAccAddr, Bech32PrefixAccPub) - cfg.SetBech32PrefixForValidator(Bech32PrefixValAddr, Bech32PrefixValPub) - cfg.SetBech32PrefixForConsensusNode(Bech32PrefixConsAddr, Bech32PrefixConsPub) - cfg.SetFullFundraiserPath(config.FullFundraiserPath) - cfg.SetCoinType(config.CosmosCoinType) + cfg.SetBech32PrefixForAccount(bech32PrefixAccAddr, bech32PrefixAccPub) + cfg.SetBech32PrefixForValidator(bech32PrefixValAddr, bech32PrefixValPub) + cfg.SetBech32PrefixForConsensusNode(bech32PrefixConsAddr, bech32PrefixConsPub) + cfg.SetFullFundraiserPath(FullFundraiserPath) + cfg.SetCoinType(CosmosCoinType) cfg.Seal() } - -// GasAdjustment is a multiplier to make sure transactions have enough gas when gas are estimated. -const GasAdjustment = 1.5 diff --git a/cosmos/genesis.go b/cosmos/genesis.go deleted file mode 100644 index 2b7b5b313..000000000 --- a/cosmos/genesis.go +++ /dev/null @@ -1,162 +0,0 @@ -package cosmos - -import ( - "encoding/json" - "os" - "time" - - "github.com/cosmos/cosmos-sdk/codec" - sdktypes "github.com/cosmos/cosmos-sdk/types" - authutils "github.com/cosmos/cosmos-sdk/x/auth/client/utils" - authexported "github.com/cosmos/cosmos-sdk/x/auth/exported" - authtypes "github.com/cosmos/cosmos-sdk/x/auth/types" - "github.com/cosmos/cosmos-sdk/x/genutil" - stakingtypes "github.com/cosmos/cosmos-sdk/x/staking/types" - "github.com/tendermint/tendermint/crypto" - "github.com/tendermint/tendermint/p2p" - "github.com/tendermint/tendermint/privval" - "github.com/tendermint/tendermint/types" - tmtypes "github.com/tendermint/tendermint/types" -) - -// GenesisValidator holds the info of a specific validator to use to generate a genesis. -type GenesisValidator struct { - Name string - Password string - ValPubKey crypto.PubKey - NodeID p2p.ID -} - -// NewGenesisValidator creates a new validator with an cosmos account, validator and node identity. -func NewGenesisValidator(kb *Keybase, name, password, privValidatorKeyFile, privValidatorStateFile, nodeKeyFile string) (GenesisValidator, error) { - val := privval.LoadOrGenFilePV(privValidatorKeyFile, privValidatorStateFile) - nodeKey, err := p2p.LoadOrGenNodeKey(nodeKeyFile) - if err != nil { - return GenesisValidator{}, err - } - return GenesisValidator{ - Name: name, - Password: password, - ValPubKey: val.GetPubKey(), - NodeID: nodeKey.ID(), - }, nil -} - -// GenesisExist returns true if the genesis file already exists. -func GenesisExist(genesisFile string) bool { - _, err := os.Stat(genesisFile) - return !os.IsNotExist(err) -} - -// LoadGenesis loads a genesis from a file. -func LoadGenesis(genesisFile string) (*tmtypes.GenesisDoc, error) { - return tmtypes.GenesisDocFromFile(genesisFile) -} - -// GenGenesis generates a new genesis and save it. -func GenGenesis(cdc *codec.Codec, kb *Keybase, defaultGenesisŚtate map[string]json.RawMessage, chainID, initialBalances, genesisFile string, validators []GenesisValidator) (*tmtypes.GenesisDoc, error) { - msgs := []sdktypes.Msg{} - for _, validator := range validators { - // get account - acc, err := kb.Get(validator.Name) - if err != nil { - return nil, err - } - // generate msg to add this validator - msgs = append(msgs, genCreateValidatorMsg(acc.GetAddress(), validator.Name, validator.ValPubKey)) - } - // generate genesis transaction - accNumber := uint64(0) - sequence := uint64(0) - b := authtypes.NewTxBuilder( - authutils.GetTxEncoder(cdc), - accNumber, - sequence, - 0, - 0, - false, - chainID, - "", - nil, - nil, - ).WithKeybase(kb) - signedMsg, err := b.BuildSignMsg(msgs) - if err != nil { - return nil, err - } - validatorTx := authtypes.NewStdTx(signedMsg.Msgs, signedMsg.Fee, nil, signedMsg.Memo) - for _, validator := range validators { - validatorTx, err = b.SignStdTx(validator.Name, validator.Password, validatorTx, true) - if err != nil { - return nil, err - } - } - // generate genesis - appState, err := genGenesisAppState(cdc, defaultGenesisŚtate, validatorTx, initialBalances) - if err != nil { - return nil, err - } - genesis, err := genGenesisDoc(cdc, appState, chainID, time.Now()) - if err != nil { - return nil, err - } - // save genesis - if err := genesis.SaveAs(genesisFile); err != nil { - return nil, err - } - return genesis, nil -} - -func genGenesisDoc(cdc *codec.Codec, appState map[string]json.RawMessage, chainID string, genesisTime time.Time) (*tmtypes.GenesisDoc, error) { - appStateEncoded, err := cdc.MarshalJSON(appState) - if err != nil { - return nil, err - } - genesis := &types.GenesisDoc{ - GenesisTime: genesisTime, - ChainID: chainID, - ConsensusParams: types.DefaultConsensusParams(), - AppState: appStateEncoded, - } - return genesis, genesis.ValidateAndComplete() -} - -func genGenesisAppState(cdc *codec.Codec, defaultGenesisŚtate map[string]json.RawMessage, signedStdTx authtypes.StdTx, initialBalances string) (map[string]json.RawMessage, error) { - genAccs := authexported.GenesisAccounts{} - pubkeys := signedStdTx.GetPubKeys() - for i, signer := range signedStdTx.GetSigners() { - initialB, err := sdktypes.ParseCoins(initialBalances) - if err != nil { - return nil, err - } - genAcc := authtypes.NewBaseAccount(signer, initialB, pubkeys[i], 0, 0) - if err := genAcc.Validate(); err != nil { - return nil, err - } - genAccs = append(genAccs, genAcc) - } - genstate, err := cdc.MarshalJSON(authtypes.NewGenesisState(authtypes.DefaultParams(), genAccs)) - if err != nil { - return nil, err - } - defaultGenesisŚtate[authtypes.ModuleName] = genstate - return genutil.SetGenTxsInAppGenesisState(cdc, defaultGenesisŚtate, []authtypes.StdTx{signedStdTx}) -} - -func genCreateValidatorMsg(accAddress sdktypes.AccAddress, accName string, valPubKey crypto.PubKey) stakingtypes.MsgCreateValidator { - return stakingtypes.NewMsgCreateValidator( - sdktypes.ValAddress(accAddress), - valPubKey, - sdktypes.NewCoin(sdktypes.DefaultBondDenom, sdktypes.TokensFromConsensusPower(100)), - stakingtypes.Description{ - Moniker: accName, - Details: "init-validator", - }, - stakingtypes.NewCommissionRates( - sdktypes.ZeroDec(), - sdktypes.ZeroDec(), - sdktypes.ZeroDec(), - ), - sdktypes.NewInt(1), - ) -} diff --git a/cosmos/genesis_test.go b/cosmos/genesis_test.go deleted file mode 100644 index 3c331af7e..000000000 --- a/cosmos/genesis_test.go +++ /dev/null @@ -1,78 +0,0 @@ -package cosmos - -import ( - "encoding/json" - "io/ioutil" - "os" - "path/filepath" - "testing" - - "github.com/cosmos/cosmos-sdk/codec" - "github.com/cosmos/cosmos-sdk/crypto/keys" - sdk "github.com/cosmos/cosmos-sdk/types" - "github.com/cosmos/cosmos-sdk/x/auth" - "github.com/cosmos/cosmos-sdk/x/staking" - "github.com/stretchr/testify/require" -) - -func TestGenesis(t *testing.T) { - // codec - cdc := codec.New() - sdk.RegisterCodec(cdc) - codec.RegisterCrypto(cdc) - auth.RegisterCodec(cdc) - staking.RegisterCodec(cdc) - - // path - path, _ := ioutil.TempDir("", "TestGenesis") - defer os.RemoveAll(path) - // keybase - kb, err := NewKeybase(filepath.Join(path, "kb")) - require.NoError(t, err) - // variables - var ( - chainID = "test-chainID" - initialBalances = "100amesg" - name = "name" - password = "pass" - privValidatorKeyFile = filepath.Join(path, "privValidatorKeyFile.json") - privValidatorStateFile = filepath.Join(path, "privValidatorStateFile.json") - nodeKeyFile = filepath.Join(path, "nodeKeyFile.json") - genesisPath = filepath.Join(path, "genesis.json") - validators = []GenesisValidator{} - defaultGenesisState = map[string]json.RawMessage{} - ) - // init account - mnemonic, _ := kb.NewMnemonic() - kb.CreateAccount(name, mnemonic, "", password, keys.CreateHDPath(0, 0).String(), DefaultAlgo) - // start tests - t.Run("generate validator", func(t *testing.T) { - v, err := NewGenesisValidator(kb, name, password, privValidatorKeyFile, privValidatorStateFile, nodeKeyFile) - validators = append(validators, v) - require.NoError(t, err) - require.Equal(t, name, v.Name) - require.Equal(t, password, v.Password) - require.NotEmpty(t, v.ValPubKey) - require.NotEmpty(t, v.NodeID) - require.FileExists(t, privValidatorKeyFile) - require.FileExists(t, privValidatorStateFile) - require.FileExists(t, nodeKeyFile) - }) - t.Run("genesis doesn't exist", func(t *testing.T) { - require.False(t, GenesisExist(genesisPath)) - }) - t.Run("generate genesis", func(t *testing.T) { - genesis, err := GenGenesis(cdc, kb, defaultGenesisState, chainID, initialBalances, genesisPath, validators) - require.NoError(t, err) - require.NotEmpty(t, genesis) - }) - t.Run("load genesis", func(t *testing.T) { - genesis, err := LoadGenesis(genesisPath) - require.NoError(t, err) - require.NotEmpty(t, genesis) - require.Equal(t, chainID, genesis.ChainID) - }) - t.Run("genesis exist", func(t *testing.T) { - require.True(t, GenesisExist(genesisPath)) - }) -} diff --git a/cosmos/keybase.go b/cosmos/keybase.go index 42b3b8164..5aad28524 100644 --- a/cosmos/keybase.go +++ b/cosmos/keybase.go @@ -13,9 +13,6 @@ import ( ) const ( - // DefaultAlgo for create account. - DefaultAlgo = keys.Secp256k1 - mnemonicEntropySize = 256 ) diff --git a/cosmos/node.go b/cosmos/node.go deleted file mode 100644 index c8458f90a..000000000 --- a/cosmos/node.go +++ /dev/null @@ -1,32 +0,0 @@ -package cosmos - -import ( - bam "github.com/cosmos/cosmos-sdk/baseapp" - "github.com/mesg-foundation/engine/logger" - tmconfig "github.com/tendermint/tendermint/config" - "github.com/tendermint/tendermint/node" - "github.com/tendermint/tendermint/p2p" - "github.com/tendermint/tendermint/privval" - "github.com/tendermint/tendermint/proxy" - "github.com/tendermint/tendermint/types" -) - -// NewNode creates a new Tendermint node from an App. -func NewNode(app *bam.BaseApp, cfg *tmconfig.Config, genesis *types.GenesisDoc) (*node.Node, error) { - nodeKey, err := p2p.LoadOrGenNodeKey(cfg.NodeKeyFile()) - if err != nil { - return nil, err - } - - // init node - return node.NewNode( - cfg, - privval.LoadOrGenFilePV(cfg.PrivValidatorKeyFile(), cfg.PrivValidatorStateFile()), - nodeKey, - proxy.NewLocalClientCreator(app), - func() (*types.GenesisDoc, error) { return genesis, nil }, - node.DefaultDBProvider, - node.DefaultMetricsProvider(cfg.Instrumentation), - logger.TendermintLogger(), - ) -} diff --git a/e2e/api_test.go b/e2e/api_test.go index 57905c7a8..094731709 100644 --- a/e2e/api_test.go +++ b/e2e/api_test.go @@ -4,15 +4,12 @@ import ( "context" "encoding/base64" "fmt" - "os" - "path/filepath" "testing" "time" "github.com/cosmos/cosmos-sdk/crypto/keys" sdk "github.com/cosmos/cosmos-sdk/types" "github.com/mesg-foundation/engine/app" - "github.com/mesg-foundation/engine/config" "github.com/mesg-foundation/engine/container" "github.com/mesg-foundation/engine/cosmos" "github.com/mesg-foundation/engine/execution" @@ -28,28 +25,42 @@ type apiclient struct { RunnerClient orchestrator.RunnerClient } -var ( - minExecutionPrice sdk.Coins - client *apiclient - cdc = app.MakeCodec() - processInitialBalance = sdk.NewCoins(sdk.NewInt64Coin("atto", 10000000)) - kb *cosmos.Keybase - cfg *config.Config - engineAddress sdk.AccAddress - cont *container.Container - lcd *cosmos.LCD - lcdEngine *cosmos.LCD - cliAddress sdk.AccAddress -) - const ( - lcdEndpoint = "http://127.0.0.1:1317/" - pollingInterval = 500 * time.Millisecond // half a block - pollingTimeout = 10 * time.Second // 10 blocks - engineMnemonic = "neutral false together tattoo matrix stamp poem mouse chair chair grain pledge mandate layer shiver embark struggle vicious antenna total faith genre valley mandate" + chainID = "mesg-dev-chain" + gasPrices = "1.0atto" + pollingInterval = 500 * time.Millisecond // half a block + pollingTimeout = 10 * time.Second // 10 blocks + + lcdEndpoint = "http://127.0.0.1:1317/" + orchestratorEndpoint = "localhost:50052" + orchestratorName = "engine" + orchestratorNetwork = "engine" + + engineMnemonic = "neutral false together tattoo matrix stamp poem mouse chair chair grain pledge mandate layer shiver embark struggle vicious antenna total faith genre valley mandate" + engineAccountName = "engine" + engineAccountPassword = "pass" + engineAccountNumber = uint32(0) + engineAccountIndex = uint32(0) + cliAccountMnemonic = "spike raccoon obscure program raw large unaware dragon hamster round artist case fall wage sample velvet robust legend identify innocent film coral picture organ" cliAccountName = "cli" cliAccountPassword = "pass" + cliAccountNumber = uint32(0) + cliAccountIndex = uint32(0) +) + +var ( + executionPrice = sdk.NewCoins(sdk.NewInt64Coin("atto", 10000)) // /x/execution/internal/types/params.go#DefaultMinPrice + processInitialBalance = sdk.NewCoins(sdk.NewInt64Coin("atto", 10000000)) + + cdc = app.MakeCodec() + client *apiclient + kb *cosmos.Keybase + engineAddress sdk.AccAddress + cont *container.Container + lcd *cosmos.LCD + lcdEngine *cosmos.LCD + cliAddress sdk.AccAddress ) func TestAPI(t *testing.T) { @@ -62,39 +73,30 @@ func TestAPI(t *testing.T) { // init config var err error - cfg, err = config.New() - require.NoError(t, err) - minExecutionPrice, err = sdk.ParseCoins(cfg.DefaultExecutionPrice) - require.NoError(t, err) - - // change and recreate cosmos relative path because CI dir permissions - cfg.Cosmos.RelativePath = "e2e.cosmos" - err = os.MkdirAll(filepath.Join(cfg.Path, cfg.Cosmos.RelativePath), os.FileMode(0755)) - require.NoError(t, err) // init keybase with engine account and cli account - kb, err = cosmos.NewKeybase(filepath.Join(cfg.Path, cfg.Cosmos.RelativePath)) - require.NoError(t, err) + kb = cosmos.NewInMemoryKeybase() + // init engine account - engineAcc, err := kb.CreateAccount(cfg.Account.Name, engineMnemonic, "", cfg.Account.Password, keys.CreateHDPath(cfg.Account.Number, cfg.Account.Index).String(), cosmos.DefaultAlgo) + engineAcc, err := kb.CreateAccount(engineAccountName, engineMnemonic, "", engineAccountPassword, keys.CreateHDPath(engineAccountNumber, engineAccountIndex).String(), cosmos.DefaultAlgo) require.NoError(t, err) engineAddress = engineAcc.GetAddress() // init cli account - cliAcc, err := kb.CreateAccount(cliAccountName, cliAccountMnemonic, "", cliAccountPassword, keys.CreateHDPath(cfg.Account.Number, cfg.Account.Index).String(), cosmos.DefaultAlgo) + cliAcc, err := kb.CreateAccount(cliAccountName, cliAccountMnemonic, "", cliAccountPassword, keys.CreateHDPath(cliAccountNumber, cliAccountIndex).String(), cosmos.DefaultAlgo) require.NoError(t, err) cliAddress = cliAcc.GetAddress() // init LCD with engine account and make a transfer to cli account - lcdEngine, err = cosmos.NewLCD(lcdEndpoint, cdc, kb, cfg.DevGenesis.ChainID, cfg.Account.Name, cfg.Account.Password, cfg.Cosmos.MinGasPrices) + lcdEngine, err = cosmos.NewLCD(lcdEndpoint, cdc, kb, chainID, engineAccountName, engineAccountPassword, gasPrices) require.NoError(t, err) // init container - cont, err = container.New(cfg.Name, cfg.Server.Address, cfg.Name) + cont, err = container.New(orchestratorName, orchestratorEndpoint, orchestratorNetwork) require.NoError(t, err) - // init gRPC client - conn, err := grpc.DialContext(context.Background(), "localhost:50052", grpc.WithInsecure()) + // init orchestrator gRPC client + conn, err := grpc.DialContext(context.Background(), orchestratorEndpoint, grpc.WithInsecure()) require.NoError(t, err) client = &apiclient{ @@ -104,7 +106,7 @@ func TestAPI(t *testing.T) { } // init LCD - lcd, err = cosmos.NewLCD(lcdEndpoint, cdc, kb, cfg.DevGenesis.ChainID, cliAccountName, cliAccountPassword, cfg.Cosmos.MinGasPrices) + lcd, err = cosmos.NewLCD(lcdEndpoint, cdc, kb, chainID, cliAccountName, cliAccountPassword, gasPrices) require.NoError(t, err) // run tests diff --git a/e2e/orchestrator_balance_withdraw_test.go b/e2e/orchestrator_balance_withdraw_test.go index 9278ece38..52996778b 100644 --- a/e2e/orchestrator_balance_withdraw_test.go +++ b/e2e/orchestrator_balance_withdraw_test.go @@ -112,10 +112,10 @@ func testOrchestratorProcessBalanceWithdraw(runnerHash, instanceHash hash.Hash) t.Run("check coins on process after 1 execution", func(t *testing.T) { var coins sdk.Coins require.NoError(t, lcd.Get("bank/balances/"+procAddress.String(), &coins)) - require.True(t, coins.IsEqual(processInitialBalance.Sub(minExecutionPrice)), coins) + require.True(t, coins.IsEqual(processInitialBalance.Sub(executionPrice)), coins) }) t.Run("withdraw from process", func(t *testing.T) { - coins := minExecutionPrice + coins := executionPrice msg := ownership.MsgWithdraw{ Owner: cliAddress, Amount: coins.String(), @@ -125,7 +125,7 @@ func testOrchestratorProcessBalanceWithdraw(runnerHash, instanceHash hash.Hash) require.NoError(t, err) require.NoError(t, lcd.Get("bank/balances/"+procAddress.String(), &coins)) - require.True(t, coins.IsEqual(processInitialBalance.Sub(minExecutionPrice).Sub(minExecutionPrice)), coins) + require.True(t, coins.IsEqual(processInitialBalance.Sub(executionPrice).Sub(executionPrice)), coins) }) t.Run("delete process", func(t *testing.T) { _, err := lcd.BroadcastMsg(processmodule.MsgDelete{ diff --git a/logger/init.go b/logger/init.go deleted file mode 100644 index 7fda96a1a..000000000 --- a/logger/init.go +++ /dev/null @@ -1,27 +0,0 @@ -package logger - -import ( - "fmt" - - "github.com/sirupsen/logrus" -) - -// Init initializes default logger. It panics on invalid format or level. -func Init(format, level string, forceColors bool) { - switch format { - case "text": - logrus.SetFormatter(&logrus.TextFormatter{ - ForceColors: forceColors, - }) - case "json": - logrus.SetFormatter(&logrus.JSONFormatter{}) - default: - panic(fmt.Sprintf("log: %s is not a valid format", format)) - } - - l, err := logrus.ParseLevel(level) - if err != nil { - panic(fmt.Sprintf("log: %s is not a valid level", level)) - } - logrus.SetLevel(l) -} diff --git a/logger/tendermint.go b/logger/tendermint.go deleted file mode 100644 index e252d5f9c..000000000 --- a/logger/tendermint.go +++ /dev/null @@ -1,72 +0,0 @@ -package logger - -import ( - "errors" - "fmt" - - "github.com/sirupsen/logrus" - "github.com/tendermint/tendermint/libs/log" -) - -// logrusLogger is common interface for logrus.Logger and logrus.Entry. -type logrusLogger interface { - Debug(args ...interface{}) - Info(args ...interface{}) - Error(args ...interface{}) - WithFields(fields logrus.Fields) *logrus.Entry -} - -// tendermintLogger wraps logrus logger into tendermint logger. -type tendermintLogger struct { - logrusLogger -} - -// Debug logs a message at level Debug on the standard logger. -func (l *tendermintLogger) Debug(msg string, keyvals ...interface{}) { - if len(keyvals) == 0 { - l.logrusLogger.Debug(msg) - } else { - l.With(keyvals...).(*tendermintLogger).logrusLogger.Debug(msg) - } -} - -// Info logs a message at level Info on the standard logger. -func (l *tendermintLogger) Info(msg string, keyvals ...interface{}) { - if len(keyvals) == 0 { - l.logrusLogger.Info(msg) - } else { - l.With(keyvals...).(*tendermintLogger).logrusLogger.Info(msg) - } -} - -// Error logs a message at level Error on the standard logger. -func (l *tendermintLogger) Error(msg string, keyvals ...interface{}) { - if len(keyvals) == 0 { - l.logrusLogger.Error(msg) - } else { - l.With(keyvals...).(*tendermintLogger).logrusLogger.Error(msg) - } -} - -// With creates an entry from the logger and adds a key values to it. -func (l *tendermintLogger) With(keyvals ...interface{}) log.Logger { - fields := logrus.Fields{} - for i := 0; i < len(keyvals); i += 2 { - if i+1 < len(keyvals) { - fields[fmt.Sprint(keyvals[i])] = keyvals[i+1] - } else { - fields[fmt.Sprint(keyvals[i])] = errors.New("(MISSING)") - } - } - return &tendermintLogger{ - logrusLogger: l.logrusLogger.WithFields(fields), - } -} - -// TendermintLogger returns standard logrus logger -// wrapped into tendermint logger interface. -func TendermintLogger() log.Logger { - return &tendermintLogger{ - logrusLogger: logrus.StandardLogger(), - } -} diff --git a/scripts/dev-cli.sh b/scripts/dev-cli.sh deleted file mode 100755 index f4853e4db..000000000 --- a/scripts/dev-cli.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/bash - -# turn on bash's job control -set -m - -# start daemon -mesg-daemon start & -daemon=$! - -# wait 5 sec for the daemon to start rpc server -sleep 5 & -wait $! - -# start lcd -mesg-cli rest-server --laddr tcp://0.0.0.0:1317 & -lcd=$! - -# start orchestrator -mesg-cli orchestrator start & -orchestrator=$! - -# this variable is used to control the monitoring of the child process -monitoring=true - -# function that stop child processes -function stop { - monitoring=false - echo "stopping all child processes" - kill $daemon $lcd $orchestrator - wait $daemon $lcd $orchestrator -} - -# trap both sigint (ctrl+c) and sigterm (OS ask process to be stopped) -trap stop SIGINT -trap stop SIGTERM - -# start the monitoring loop -while $monitoring -do - # check if all sub processes are running correctly - if [[ -n "$(ps -p $daemon -o pid=)" ]] && [[ -n "$(ps -p $lcd -o pid=)" ]] && [[ -n "$(ps -p $orchestrator -o pid=)" ]]; then - sleep 2 & - wait $! - else - # if one child process is not running, stopping all of them and exit with error code 1 - echo "one child process is not running" - stop - exit 1 - fi -done diff --git a/version/version.go b/version/version.go deleted file mode 100644 index 7850e2b48..000000000 --- a/version/version.go +++ /dev/null @@ -1,4 +0,0 @@ -package version - -// Version of this release. Will be replaced automatically when compiling in CI -var Version = "vX.X.X" From 8b5f2478965b6313eb8375a3d8b136731f78b668 Mon Sep 17 00:00:00 2001 From: Nicolas Mahe Date: Wed, 6 May 2020 13:38:08 +0700 Subject: [PATCH 26/33] update dockerfile and scripts to switch all to cli based engine --- .circleci/config.yml | 26 +++---- .dockerignore | 1 - Dockerfile | 28 ++++--- Dockerfile.cli | 27 ------- Dockerfile.cli.dev | 8 -- Dockerfile.dev | 15 ++-- Makefile | 121 ++++++++++++++--------------- go.mod | 2 - scripts/build-cli.sh | 19 +++++ scripts/build-engine.sh | 40 ---------- scripts/dev-starter.sh | 50 ++++++++++++ scripts/dev.sh | 164 ++++++++++++---------------------------- scripts/publish-cli.sh | 16 ++++ scripts/publish-cmds.sh | 28 ------- scripts/run-e2e.sh | 31 +++++--- 15 files changed, 247 insertions(+), 329 deletions(-) delete mode 100644 Dockerfile.cli delete mode 100644 Dockerfile.cli.dev create mode 100755 scripts/build-cli.sh delete mode 100755 scripts/build-engine.sh create mode 100755 scripts/dev-starter.sh create mode 100755 scripts/publish-cli.sh delete mode 100755 scripts/publish-cmds.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index 6d113f9d5..1fd2dab53 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,7 +7,7 @@ run_on_machine: &run_on_machine run_on_docker: &run_on_docker docker: - - image: circleci/golang:1.13.0 + - image: circleci/golang:1.13.10 go_path: &go_path /tmp/go/path @@ -65,12 +65,12 @@ jobs: name: "Install Go" command: | sudo rm -rf /usr/local/go - curl -sSL "https://dl.google.com/go/go1.13.linux-amd64.tar.gz" | sudo tar -xz -C /usr/local/ + curl -sSL "https://dl.google.com/go/go1.13.10.linux-amd64.tar.gz" | sudo tar -xz -C /usr/local/ echo "export PATH=$PATH:/usr/local/go/bin" >> $BASH_ENV - run: docker swarm init - <<: *restore_go_cache - <<: *restore_go_path - - run: make e2e version=`echo $CIRCLE_SHA1 | cut -c1-7` + - run: make e2e - <<: *save_go_path - <<: *save_go_cache @@ -79,33 +79,33 @@ jobs: - image: golangci/golangci-lint:v1.21 steps: - checkout - - run: make lint + - run: golangci-lint - "publish_docker_dev": + "publish_docker_unstable": <<: *run_on_machine steps: - checkout - run: docker login -u $DOCKER_USER -p $DOCKER_PASS - - run: make docker-publish-dev version=`echo $CIRCLE_SHA1 | cut -c1-7` + - run: make publish-docker-unstable version=`echo $CIRCLE_SHA1 | cut -c1-7` "publish_docker_prod": <<: *run_on_machine steps: - checkout - run: docker login -u $DOCKER_USER -p $DOCKER_PASS - - run: make docker-publish version=$CIRCLE_TAG + - run: make publish-docker-prod version=$CIRCLE_TAG - "release_cli_dev": + "release_cli_unstable": <<: *run_on_docker steps: - checkout - - run: make publish-cmds version="dev build `echo $CIRCLE_SHA1 | cut -c1-7`" release-type=dev + - run: make publish version="`echo $CIRCLE_SHA1 | cut -c1-7`" release-type=unstable "release_cli_prod": <<: *run_on_docker steps: - checkout - - run: make publish-cmds version=$CIRCLE_TAG release-type=prod + - run: make publish version=$CIRCLE_TAG release-type=prod workflows: version: 2 @@ -133,7 +133,7 @@ workflows: - "test" - "lint" - release_dev: + release_unstable: jobs: - "lint": filters: @@ -151,10 +151,10 @@ workflows: requires: - "test" - "lint" - - "publish_docker_dev": + - "publish_docker_unstable": requires: - "test_e2e" - - "release_cli_dev": + - "release_cli_unstable": requires: - "test_e2e" diff --git a/.dockerignore b/.dockerignore index 3110d40ff..51b9d5e25 100644 --- a/.dockerignore +++ b/.dockerignore @@ -2,7 +2,6 @@ Dockerfile # build /bin -!/bin/engine # repo files README.md diff --git a/Dockerfile b/Dockerfile index 3eb2d100c..9b4d59217 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,21 +1,27 @@ -# base Go image version. -FROM golang:1.13.0-stretch AS build - -WORKDIR /project +# base Go image version for building the binaries +FROM golang:1.13.10 AS build +WORKDIR /app -# install dependencies COPY go.mod go.sum ./ RUN go mod download COPY . . ARG version -RUN go build -mod=readonly -o ./bin/engine -ldflags="-X 'github.com/mesg-foundation/engine/version.Version=$version'" core/main.go +RUN go build -mod=readonly -o ./bin/mesg-cli -ldflags="-s -w -X 'github.com/cosmos/cosmos-sdk/version.Name=mesg' -X 'github.com/cosmos/cosmos-sdk/version.ServerName=mesg-daemon' -X 'github.com/cosmos/cosmos-sdk/version.ClientName=mesg-cli' -X 'github.com/cosmos/cosmos-sdk/version.Version=$version'" ./cmd/mesg-cli/ +RUN go build -mod=readonly -o ./bin/mesg-daemon -ldflags="-s -w -X 'github.com/cosmos/cosmos-sdk/version.Name=mesg' -X 'github.com/cosmos/cosmos-sdk/version.ServerName=mesg-daemon' -X 'github.com/cosmos/cosmos-sdk/version.ClientName=mesg-cli' -X 'github.com/cosmos/cosmos-sdk/version.Version=$version'" ./cmd/mesg-daemon/ + +# ubuntu image with binaries for distribution FROM ubuntu:18.04 RUN apt-get update && \ - apt-get install -y --no-install-recommends ca-certificates=20180409 && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* + apt-get install -y --no-install-recommends ca-certificates=20180409 && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + WORKDIR /app -COPY --from=build /project/bin/engine . -CMD ["./engine"] +ENV PATH="/app:${PATH}" + +COPY --from=build /app/bin/mesg-cli . +COPY --from=build /app/bin/mesg-daemon . + +CMD ["mesg-daemon", "start"] diff --git a/Dockerfile.cli b/Dockerfile.cli deleted file mode 100644 index 9b4d59217..000000000 --- a/Dockerfile.cli +++ /dev/null @@ -1,27 +0,0 @@ -# base Go image version for building the binaries -FROM golang:1.13.10 AS build -WORKDIR /app - -COPY go.mod go.sum ./ -RUN go mod download - -COPY . . -ARG version - -RUN go build -mod=readonly -o ./bin/mesg-cli -ldflags="-s -w -X 'github.com/cosmos/cosmos-sdk/version.Name=mesg' -X 'github.com/cosmos/cosmos-sdk/version.ServerName=mesg-daemon' -X 'github.com/cosmos/cosmos-sdk/version.ClientName=mesg-cli' -X 'github.com/cosmos/cosmos-sdk/version.Version=$version'" ./cmd/mesg-cli/ -RUN go build -mod=readonly -o ./bin/mesg-daemon -ldflags="-s -w -X 'github.com/cosmos/cosmos-sdk/version.Name=mesg' -X 'github.com/cosmos/cosmos-sdk/version.ServerName=mesg-daemon' -X 'github.com/cosmos/cosmos-sdk/version.ClientName=mesg-cli' -X 'github.com/cosmos/cosmos-sdk/version.Version=$version'" ./cmd/mesg-daemon/ - -# ubuntu image with binaries for distribution -FROM ubuntu:18.04 -RUN apt-get update && \ - apt-get install -y --no-install-recommends ca-certificates=20180409 && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - -WORKDIR /app -ENV PATH="/app:${PATH}" - -COPY --from=build /app/bin/mesg-cli . -COPY --from=build /app/bin/mesg-daemon . - -CMD ["mesg-daemon", "start"] diff --git a/Dockerfile.cli.dev b/Dockerfile.cli.dev deleted file mode 100644 index 5be25223c..000000000 --- a/Dockerfile.cli.dev +++ /dev/null @@ -1,8 +0,0 @@ -ARG from -FROM $from - -COPY ./dev-chain/cli /root/.mesg-cli -COPY ./dev-chain/validator /root/.mesg-node -COPY ./scripts/dev-cli.sh . - -CMD ["bash", "dev-cli.sh"] diff --git a/Dockerfile.dev b/Dockerfile.dev index 4387a7945..046be1655 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -1,7 +1,8 @@ -FROM ubuntu:18.04 -RUN apt-get update && \ - apt-get install -y --no-install-recommends ca-certificates=20180409 && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* -COPY ./bin/engine . -CMD [ "./engine" ] +ARG from +FROM $from + +COPY ./dev-chain/cli /root/.mesg-cli +COPY ./dev-chain/validator /root/.mesg-node +COPY ./scripts/dev-starter.sh . + +CMD ["bash", "dev-starter.sh"] diff --git a/Makefile b/Makefile index eee6cc5b6..ec2d25e0a 100644 --- a/Makefile +++ b/Makefile @@ -1,98 +1,87 @@ -.PHONY: all build build-cmd-cosmos changelog check-version clean clean-build clean-docker dep dev dev-mon dev-start dev-stop docker-build docker-dev docker-publish docker-publish-dev docker-tools genesis lint protobuf test publish-cmds build-docker-cli build-docker-cli-dev +.PHONY: build build-docker publish publish-docker-prod publish-docker-unstable test e2e dep lint build-tools protobuf changelog clean dev dev-mon +version ?= local MAJOR_VERSION := $(shell echo $(version) | cut -d . -f 1) MINOR_VERSION := $(shell echo $(version) | cut -d . -f 1-2) -PATCH_VERSION := $(version) -all: clean lint build test e2e +# Build -check-version: -ifndef version - $(error version is undefined) -endif +build: dep + ./scripts/build-cli.sh "$(version)" -docker-build: check-version +build-docker: docker build \ - --build-arg version=$(PATCH_VERSION) \ - -t mesg/engine:$(MAJOR_VERSION) \ - -t mesg/engine:$(MINOR_VERSION) \ - -t mesg/engine:$(PATCH_VERSION) \ - -t mesg/engine:latest \ + --build-arg version=$(version) \ + -t mesg/engine:$(version) \ + . + docker build \ + -f ./Dockerfile.dev \ + --build-arg from=mesg/engine:$(version) \ + -t mesg/engine:$(version)-dev \ . -docker-dev: dep - ./scripts/build-engine.sh - -docker-publish: docker-build - docker push mesg/engine:$(MAJOR_VERSION) - docker push mesg/engine:$(MINOR_VERSION) - docker push mesg/engine:$(PATCH_VERSION) - docker push mesg/engine:latest +# Publish -docker-publish-dev: check-version - docker build -t mesg/engine:dev --build-arg version=$(version) . - docker push mesg/engine:dev +publish: build + ./scripts/publish-cli.sh "$(version)" "$(release-type)" -docker-tools: - docker build -t mesg/tools:local -f Dockerfile.tools . +publish-docker-prod: build-docker + docker tag mesg/engine:$(version) mesg/engine:$(MINOR_VERSION) + docker tag mesg/engine:$(version) mesg/engine:$(MAJOR_VERSION) + docker tag mesg/engine:$(version) mesg/engine:latest -dev: docker-dev - - ./scripts/dev.sh + docker push mesg/engine:$(version) + docker push mesg/engine:$(MINOR_VERSION) + docker push mesg/engine:$(MAJOR_VERSION) + docker push mesg/engine:latest -dev-mon: docker-dev - - ./scripts/dev.sh -m + docker tag mesg/engine:$(version)-dev mesg/engine:$(MINOR_VERSION)-dev + docker tag mesg/engine:$(version)-dev mesg/engine:$(MAJOR_VERSION)-dev + docker tag mesg/engine:$(version)-dev mesg/engine:latest-dev -dev-start: docker-dev - ./scripts/dev.sh -q + docker push mesg/engine:$(version)-dev + docker push mesg/engine:$(MINOR_VERSION)-dev + docker push mesg/engine:$(MAJOR_VERSION)-dev + docker push mesg/engine:latest-dev -dev-stop: docker-dev - ./scripts/dev.sh -m stop +publish-docker-unstable: build-docker + docker tag mesg/engine:$(version) mesg/engine:unstable + docker push mesg/engine:unstable -dep: - go mod download + docker tag mesg/engine:$(version)-dev mesg/engine:unstable-dev + docker push mesg/engine:unstable-dev -build: check-version dep - go build -mod=readonly -o ./bin/engine -ldflags="-X 'github.com/mesg-foundation/engine/version.Version=$(version)'" core/main.go +# Test -publish-cmds: check-version dep - ./scripts/publish-cmds.sh "$(version)" "$(release-type)" +test: dep + go test -short -mod=readonly -v -coverprofile=coverage.txt ./... -build-cmd: dep - go build -mod=readonly -o ./bin/mesg-cli ./cmd/mesg-cli/ - go build -mod=readonly -o ./bin/mesg-daemon ./cmd/mesg-daemon/ +e2e: build-docker + ./scripts/run-e2e.sh "$(version)" -build-docker-cli: check-version - docker build -t mesg/engine:cli -f ./Dockerfile.cli --build-arg version=$(version) . +dev: build-docker + ./scripts/dev.sh "$(version)" -build-docker-cli-dev: build-docker-cli - docker build -t mesg/engine:cli-dev -f ./Dockerfile.cli.dev --build-arg from=mesg/engine:cli . +dev-mon: build-docker + ./scripts/dev.sh "$(version)" "monitoring" -e2e: build-docker-cli-dev - ./scripts/run-e2e.sh +# MISC -test: dep - go test -short -mod=readonly -v -coverprofile=coverage.txt ./... +dep: + go mod download lint: - golangci-lint run + docker run --rm -v $(PWD):/app -w /app golangci/golangci-lint:v1.21 golangci-lint run -protobuf: docker-tools +build-tools: + docker build -t mesg/tools:local -f Dockerfile.tools . + +protobuf: build-tools docker run --rm -v $(PWD):/project mesg/tools:local ./scripts/build-proto.sh changelog: ./scripts/changelog.sh $(milestone) -clean-build: +clean: - rm -rf bin - -clean-docker: - - docker image rm \ - mesg/engine:$(version) \ - mesg/engine:latest \ - mesg/engine:local \ - mesg/engine:dev 2>/dev/null - -clean: clean-build clean-docker - -genesis: - go run internal/tools/gen-genesis/main.go --path $(path) --chain-id $(chain-id) --validators $(validators) + - docker image rm $(docker images | grep 'mesg') diff --git a/go.mod b/go.mod index 09597946c..e1134b598 100644 --- a/go.mod +++ b/go.mod @@ -32,7 +32,6 @@ require ( github.com/huandu/xstrings v1.2.0 // indirect github.com/imdario/mergo v0.3.7 // indirect github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect - github.com/mitchellh/go-homedir v1.1.0 github.com/morikuni/aec v1.0.0 // indirect github.com/mr-tron/base58 v1.1.3 github.com/mwitkow/go-proto-validators v0.0.0-20190212092829-1f388280e944 // indirect @@ -57,7 +56,6 @@ require ( golang.org/x/net v0.0.0-20200301022130-244492dfa37a // indirect golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f // indirect google.golang.org/grpc v1.29.1 - gopkg.in/yaml.v2 v2.2.8 gotest.tools v2.2.0+incompatible // indirect ) diff --git a/scripts/build-cli.sh b/scripts/build-cli.sh new file mode 100755 index 000000000..8f35c177f --- /dev/null +++ b/scripts/build-cli.sh @@ -0,0 +1,19 @@ +#!/bin/bash -e + +if [[ -z "$1" ]]; then + echo -e "version is not set, run:\n" + echo "$0 vX.X.X" + exit 1 +fi + +LDFLAGS="-s -w -X 'github.com/cosmos/cosmos-sdk/version.Name=mesg' -X 'github.com/cosmos/cosmos-sdk/version.ServerName=mesg-daemon' -X 'github.com/cosmos/cosmos-sdk/version.ClientName=mesg-cli' -X 'github.com/cosmos/cosmos-sdk/version.Version=$1'" +oss=(linux darwin) +archs=(amd64 386) + +for os in ${oss[*]}; do + for arch in ${archs[*]}; do + echo "Building $os $arch..." + CGO_ENABLED=0 GOOS=$os GOARCH=$arch go build -mod=readonly -o "./bin/mesg-daemon-$os-$arch" -ldflags="$LDFLAGS" ./cmd/mesg-daemon/ + CGO_ENABLED=0 GOOS=$os GOARCH=$arch go build -mod=readonly -o "./bin/mesg-cli-$os-$arch" -ldflags="$LDFLAGS" ./cmd/mesg-cli/ + done +done diff --git a/scripts/build-engine.sh b/scripts/build-engine.sh deleted file mode 100755 index 4ee4e9c93..000000000 --- a/scripts/build-engine.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash - -# script is use to use local go cache to speed up build -# docker dosen't allow to mount volume during build, -# so it always rebuild whole project. Use go build cache to -# make docker build faster. - -set -e - -ENGINE_SUM_PATH="./bin/.engine.sum" -DOCKER_SUM_PATH="./bin/.Dockerfile.dev.sum" - -echo "compile engine" -GOOS=linux GOARCH=amd64 go build -o ./bin/engine core/main.go - -touch "$ENGINE_SUM_PATH" "$DOCKER_SUM_PATH" - -# check if engine bin was cached -ENGINE_SUM="$(openssl md5 ./bin/engine)" -ENGINE_SUM_PREV="$(cat $ENGINE_SUM_PATH)" -if [[ "$ENGINE_SUM" == "$ENGINE_SUM_PREV" ]]; then - BINCACHED=true -else - echo "$ENGINE_SUM" > "$ENGINE_SUM_PATH" -fi - -# check if dockerfile was cached -DOCKER_SUM="$(openssl md5 ./Dockerfile.dev)" -DOCKER_SUM_PREV="$(cat $DOCKER_SUM_PATH)" -if [[ "$DOCKER_SUM" == "$DOCKER_SUM_PREV" ]]; then - DOCKERCACHED=true -else - echo "$DOCKER_SUM" > "$DOCKER_SUM_PATH" -fi - -IMAGE_EXIST="$(docker image ls mesg/engine:local -q)" -if [[ ! $BINCACHED ]] || [[ ! $DOCKERCACHED ]] || [[ $IMAGE_EXIST == "" ]]; then - echo "build mesg/engine image" - docker build -f Dockerfile.dev -t "mesg/engine:local" . -fi diff --git a/scripts/dev-starter.sh b/scripts/dev-starter.sh new file mode 100755 index 000000000..f4853e4db --- /dev/null +++ b/scripts/dev-starter.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +# turn on bash's job control +set -m + +# start daemon +mesg-daemon start & +daemon=$! + +# wait 5 sec for the daemon to start rpc server +sleep 5 & +wait $! + +# start lcd +mesg-cli rest-server --laddr tcp://0.0.0.0:1317 & +lcd=$! + +# start orchestrator +mesg-cli orchestrator start & +orchestrator=$! + +# this variable is used to control the monitoring of the child process +monitoring=true + +# function that stop child processes +function stop { + monitoring=false + echo "stopping all child processes" + kill $daemon $lcd $orchestrator + wait $daemon $lcd $orchestrator +} + +# trap both sigint (ctrl+c) and sigterm (OS ask process to be stopped) +trap stop SIGINT +trap stop SIGTERM + +# start the monitoring loop +while $monitoring +do + # check if all sub processes are running correctly + if [[ -n "$(ps -p $daemon -o pid=)" ]] && [[ -n "$(ps -p $lcd -o pid=)" ]] && [[ -n "$(ps -p $orchestrator -o pid=)" ]]; then + sleep 2 & + wait $! + else + # if one child process is not running, stopping all of them and exit with error code 1 + echo "one child process is not running" + stop + exit 1 + fi +done diff --git a/scripts/dev.sh b/scripts/dev.sh index f24ecbcf8..4f37143fa 100755 --- a/scripts/dev.sh +++ b/scripts/dev.sh @@ -2,133 +2,65 @@ set -e -# mesg config variables -MESG_NAME=${MESG_NAME:-"engine"} -MESG_PATH=${MESG_PATH:-"$HOME/.mesg"} +ENGINE_NAME="engine" +NETWORK_NAME="engine" -MESG_SERVER_PORT=${MESG_SERVER_PORT:-"50052"} -MESG_TENDERMINT_NETWORK="mesg-tendermint" -MESG_TENDERMINT_PORT=${MESG_TENDERMINT_PORT:-"26656"} -MESG_TENDERMINT_RPC_PORT=${MESG_TENDERMINT_RPC_PORT:-"26657"} -MESG_PROMETHEUS_RPC_PORT=${MESG_PROMETHEUS_RPC_PORT:-"26660"} -MESG_COSMOS_RPC_PORT=${MESG_COSMOS_RPC_PORT:-"1317"} +if [[ -z "$1" ]]; then + echo -e "arg version is missing, run:\n" + echo "$0 vX.X.X" + exit 1 +fi -# cmd args -quiet=false -monitor=false +monitoring=false +if [[ "$2" == "monitoring" ]]; then + monitoring=true +fi function onexit { - set +e - echo -e "\nshutting down, please wait..." - docker_service_remove "$MESG_NAME" + docker service rm $ENGINE_NAME + docker wait $(docker ps -f label=com.docker.swarm.service.name=$ENGINE_NAME -q) 2> /dev/null + if $monitor; then docker service rm engine-grafana engine-prometheus + docker wait $(docker ps -f label=com.docker.swarm.service.name=engine-grafana -q) 2> /dev/null + docker wait $(docker ps -f label=com.docker.swarm.service.name=engine-prometheus -q) 2> /dev/null fi - docker_network_remove "$MESG_NAME" - docker_network_remove "$MESG_TENDERMINT_NETWORK" -} - -function docker_service_remove { - docker service rm $1 - docker wait $(docker ps -f label=com.docker.swarm.service.name=$1 -q) 2> /dev/null -} -function docker_network_exist { - [[ ! -z $(docker network list -f name="$1" -f driver=overlay -q) ]] + docker network remove $NETWORK_NAME } -function docker_network_create { - echo -ne "create docker network $1: " - docker network create --driver overlay "$1" --label com.docker.stack.namespace="$1" -} +trap onexit EXIT -function docker_network_remove { - echo -ne "remove docker network: " - docker network remove "$1" -} +if [[ -z $(docker network list -f name="$NETWORK_NAME" -q) ]]; then + docker network create --driver overlay $NETWORK_NAME +fi -function start_engine { - if ! docker_network_exist "$MESG_NAME"; then - docker_network_create "$MESG_NAME" - fi - - if ! docker_network_exist "$MESG_TENDERMINT_NETWORK"; then - docker_network_create "$MESG_TENDERMINT_NETWORK" - fi - - if $monitor; then - docker service create \ - -p 3001:3000 \ - --network=engine \ - --name=engine-grafana \ - --mount type=bind,source=$(pwd)/scripts/monitoring/datasource.yml,destination=/etc/grafana/provisioning/datasources/datasource.yml \ - --mount type=bind,source=$(pwd)/scripts/monitoring/dashboard.yml,destination=/etc/grafana/provisioning/dashboards/dashboard.yml \ - --mount type=bind,source=$(pwd)/scripts/monitoring/dashboards,destination=/var/lib/grafana/dashboards \ - grafana/grafana - - docker service create \ - -p 9090:9090 \ - --network=engine \ - --name=engine-prometheus \ - --mount type=bind,source=$(pwd)/scripts/monitoring/prometheus.yml,destination=/etc/prometheus/prometheus.yml \ - prom/prometheus - fi - - mkdir -p $MESG_PATH - - echo "create docker service: " +if $monitoring; then + echo "start monitoring" docker service create \ - --name $MESG_NAME \ - --tty \ - --label com.docker.stack.namespace=$MESG_NAME \ - --label com.docker.stack.image=mesg/engine:local \ - --env MESG_NAME=$MESG_NAME \ - --mount type=bind,source=$MESG_PATH,destination=/root/.mesg \ - --network $MESG_NAME \ - --network name=$MESG_TENDERMINT_NETWORK \ - --publish $MESG_SERVER_PORT:50052 \ - --publish $MESG_TENDERMINT_PORT:26656 \ - --publish $MESG_TENDERMINT_RPC_PORT:26657 \ - --publish $MESG_PROMETHEUS_RPC_PORT:26660 \ - --publish $MESG_COSMOS_RPC_PORT:1317 \ - --stop-grace-period 1m \ - mesg/engine:local -} + -p 3001:3000 \ + --network $NETWORK_NAME \ + --name=engine-grafana \ + --mount type=bind,source=$(pwd)/scripts/monitoring/datasource.yml,destination=/etc/grafana/provisioning/datasources/datasource.yml \ + --mount type=bind,source=$(pwd)/scripts/monitoring/dashboard.yml,destination=/etc/grafana/provisioning/dashboards/dashboard.yml \ + --mount type=bind,source=$(pwd)/scripts/monitoring/dashboards,destination=/var/lib/grafana/dashboards \ + grafana/grafana -function stop_engine { - onexit -} - -while getopts "qm" o; do - case $o in - q) - quiet=true - ;; - m) - monitor=true - ;; - *) - echo "unknown flag $0" - exit 1 - ;; - esac -done -shift $((OPTIND-1)) - -cmd=${1:-"start"} - -case $cmd in - start) - start_engine - if ! $quiet; then - trap onexit EXIT - docker service logs --tail 1000 --follow --raw $MESG_NAME - fi - ;; - stop) - stop_engine - ;; - *) - echo "unknown command $cmd" - exit 1 -esac + docker service create \ + -p 9090:9090 \ + --network $NETWORK_NAME \ + --name=engine-prometheus \ + --mount type=bind,source=$(pwd)/scripts/monitoring/prometheus.yml,destination=/etc/prometheus/prometheus.yml \ + prom/prometheus +fi + +docker service create \ + --name $ENGINE_NAME \ + -p 1317:1317 \ + -p 50052:50052 \ + -p 26657:26657 \ + --network $NETWORK_NAME \ + --label com.docker.stack.namespace=$ENGINE_NAME \ + mesg/engine:$1-dev + +docker service logs --tail 1000 --follow --raw $ENGINE_NAME diff --git a/scripts/publish-cli.sh b/scripts/publish-cli.sh new file mode 100755 index 000000000..842de1fea --- /dev/null +++ b/scripts/publish-cli.sh @@ -0,0 +1,16 @@ +#!/bin/bash -e + +if [[ -z "$1" || -z "$2" || ( "$2" != "unstable" && "$2" != "prod" ) ]]; then + echo -e "version and release type are not set correctly, run:\n" + echo "$0 vX.X.X prod|unstable" + exit 1 +fi + +go install github.com/tcnksm/ghr + +if [[ "$2" == "unstable" ]]; then + ghr -u mesg-foundation -r engine -p 1 -delete -prerelease -n "Unstable release" -b "Warning - this is an unstable release, use it only if you know what are doing." unstable ./bin +fi +if [[ "$2" == "prod" ]]; then + ghr -u mesg-foundation -r engine -p 1 -replace "$1" ./bin +fi diff --git a/scripts/publish-cmds.sh b/scripts/publish-cmds.sh deleted file mode 100755 index 19a35a14a..000000000 --- a/scripts/publish-cmds.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -e - -if [[ -z "$1" || -z "$2" || ( "$2" != "dev" && "$2" != "prod" ) ]]; then - echo -e "version and release type are not set or not correct, run:\n" - echo "$0 vX.X.X prod|dev" - exit 1 -fi - -LDFLAGS="-s -w -X 'github.com/mesg-foundation/engine/version.Version=$1' -X 'github.com/cosmos/cosmos-sdk/version.Name=mesg' -X 'github.com/cosmos/cosmos-sdk/version.ServerName=mesg-daemon' -X 'github.com/cosmos/cosmos-sdk/version.ClientName=mesg-cli' -X 'github.com/cosmos/cosmos-sdk/version.Version=$1'" -archs=(amd64 386) -oss=(linux darwin) - -for os in ${oss[*]}; do - for arch in ${archs[*]}; do - echo "Building $os $arch..." - CGO_ENABLED=0 GOOS=$os GOARCH=$arch go build -mod=readonly -o "./bin/mesg-daemon-$os-$arch" -ldflags="$LDFLAGS" ./cmd/mesg-daemon/ - CGO_ENABLED=0 GOOS=$os GOARCH=$arch go build -mod=readonly -o "./bin/mesg-cli-$os-$arch" -ldflags="$LDFLAGS" ./cmd/mesg-cli/ - done -done - -go install github.com/tcnksm/ghr - -if [[ "$2" == "dev" ]]; then - ghr -u mesg-foundation -r engine -p 1 -delete -prerelease -n "Developer Release" -b "Warning - this is a developer release, use it only if you know what are doing. Make sure to pull the latest \`mesg/engine:dev\` image." release-dev ./bin -fi -if [[ "$2" == "prod" ]]; then - ghr -u mesg-foundation -r engine -p 1 -replace "$1" ./bin -fi diff --git a/scripts/run-e2e.sh b/scripts/run-e2e.sh index 1ad197165..0c07d971e 100755 --- a/scripts/run-e2e.sh +++ b/scripts/run-e2e.sh @@ -2,31 +2,42 @@ set -e +ENGINE_NAME="engine" +NETWORK_NAME="engine" + +if [[ -z "$1" ]]; then + echo -e "arg version is missing, run:\n" + echo "$0 vX.X.X" + exit 1 +fi + + echo "run non existing test to detect compilation error quickly" go test -mod=readonly -v -count=1 ./e2e/... -run=__NONE__ function onexit { - docker service rm engine - docker wait $(docker ps -f label=com.docker.swarm.service.name=engine -q) 2> /dev/null + docker service rm $ENGINE_NAME + docker wait $(docker ps -f label=com.docker.swarm.service.name=$ENGINE_NAME -q) 2> /dev/null - docker network remove engine + docker network remove $NETWORK_NAME } trap onexit EXIT -if [[ -z $(docker network list -f name="engine" -q) ]]; then - docker network create --driver overlay engine +if [[ -z $(docker network list -f name="$NETWORK_NAME" -q) ]]; then + docker network create --driver overlay $NETWORK_NAME fi + docker service create \ - --name engine \ + --name $ENGINE_NAME \ -p 1317:1317 \ -p 50052:50052 \ -p 26657:26657 \ - --network engine \ - --label com.docker.stack.namespace=engine \ - mesg/engine:cli-dev + --network $NETWORK_NAME \ + --label com.docker.stack.namespace=$ENGINE_NAME \ + mesg/engine:$1-dev -echo "waiting to give some time to the container to start and run" +echo "waiting rpc server to start" sleep 10 & wait $! From 4b6f019b6d847e25dbb3d6b02592e61687111e8b Mon Sep 17 00:00:00 2001 From: Nicolas Mahe Date: Fri, 8 May 2020 11:36:10 +0700 Subject: [PATCH 27/33] poll instead of sleep in run-e2e script --- scripts/run-e2e.sh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/scripts/run-e2e.sh b/scripts/run-e2e.sh index 1ad197165..7fbb37707 100755 --- a/scripts/run-e2e.sh +++ b/scripts/run-e2e.sh @@ -27,8 +27,14 @@ docker service create \ mesg/engine:cli-dev echo "waiting to give some time to the container to start and run" -sleep 10 & -wait $! +while true; do + printf '.' + block=$(curl --silent http://localhost:1317/node_info | jq .node_info.protocol_version.block) + if [[ -n $block ]]; then + break + fi + sleep 1 +done echo "starting tests" go test -failfast -mod=readonly -v -count=1 ./e2e/... From 6db52314b4120229b182fc39960b751cc5a18ea3 Mon Sep 17 00:00:00 2001 From: Nicolas Mahe Date: Fri, 8 May 2020 17:15:21 +0700 Subject: [PATCH 28/33] update e2e test to use docker container instead of docker service --- container/container.go | 467 +++++++++++------------------------------ go.mod | 2 +- scripts/run-e2e.sh | 11 +- 3 files changed, 124 insertions(+), 356 deletions(-) diff --git a/container/container.go b/container/container.go index 1092e4837..d17e4861e 100644 --- a/container/container.go +++ b/container/container.go @@ -4,24 +4,22 @@ import ( "context" "encoding/base64" "errors" - "fmt" "io/ioutil" "net" "net/http" "os" "strconv" - "strings" "time" "github.com/docker/cli/cli/command/image/build" "github.com/docker/docker/api/types" - "github.com/docker/docker/api/types/filters" - "github.com/docker/docker/api/types/mount" - "github.com/docker/docker/api/types/swarm" + containertypes "github.com/docker/docker/api/types/container" + mounttypes "github.com/docker/docker/api/types/mount" + networktypes "github.com/docker/docker/api/types/network" "github.com/docker/docker/client" - "github.com/docker/docker/errdefs" "github.com/docker/docker/pkg/archive" "github.com/docker/docker/pkg/idtools" + "github.com/docker/go-connections/nat" "github.com/mesg-foundation/engine/ext/xerrors" "github.com/mesg-foundation/engine/ext/xnet" "github.com/mesg-foundation/engine/ext/xos" @@ -33,37 +31,13 @@ const ( // defaultStopGracePeriod is the timeout value between stopping a container and killing it. defaultStopGracePeriod = 10 * time.Second - defaultMaxAttempts = uint64(3) + // Number of time to restart the container if it crashes. + defaultMaximumRetryCount = 3 servicePrefix = "mesg_srv_" imageTag = "mesg:" - - pollingTime = 500 * time.Millisecond -) - -// Status of the service. -type Status uint - -// Possible status for services. -const ( - UNKNOWN Status = iota - STOPPED - STARTING - RUNNING ) -// statuses is a struct used to map service and container statuses. -var statuses = []struct { - container bool - service bool - status Status -}{ - {service: true, container: true, status: RUNNING}, - {service: true, container: false, status: STARTING}, - {service: false, container: true, status: RUNNING}, // This is actually stopping - {service: false, container: false, status: STOPPED}, -} - // Container starts and stops the MESG Service in Docker Container. type Container struct { // client is a Docker client. @@ -153,6 +127,14 @@ func (c *Container) Build(srvHash hash.Hash, path string) error { // Start starts the service. func (c *Container) Start(srv *service.Service, instanceHash, runnerHash, instanceEnvHash hash.Hash, instanceEnv []string, registerPayload []byte) (err error) { + // delete the service's container on any error + errorOccurred := true + defer func() { + if errorOccurred { + c.Stop(srv, runnerHash) + } + }() + runnerName := servicePrefix + runnerHash.String() networkID, err := c.createNetwork(runnerName) @@ -164,8 +146,6 @@ func (c *Container) Start(srv *service.Service, instanceHash, runnerHash, instan return err } - specs := make([]swarm.ServiceSpec, 0) - // Create dependency container configs for _, dep := range srv.Dependencies { depName := runnerName + "_" + dep.Key @@ -174,115 +154,105 @@ func (c *Container) Start(srv *service.Service, instanceHash, runnerHash, instan if err != nil { return err } - stopGracePeriod := defaultStopGracePeriod - maxAttempts := defaultMaxAttempts - specs = append(specs, swarm.ServiceSpec{ - Annotations: swarm.Annotations{ - Name: depName, - Labels: map[string]string{ - "mesg.engine": c.engineName, - "mesg.service": srv.Hash.String(), - "mesg.instance": instanceHash.String(), - "mesg.runner": runnerHash.String(), - "mesg.dependency": dep.Key, - "com.docker.stack.namespace": depName, - }, - }, - TaskTemplate: swarm.TaskSpec{ - ContainerSpec: &swarm.ContainerSpec{ - Image: dep.Image, - Labels: map[string]string{ - "com.docker.stack.namespace": depName, - }, - Env: dep.Env, - Args: dep.Args, - Command: strings.Fields(dep.Command), - Mounts: append(volumes, volumesFrom...), - StopGracePeriod: &stopGracePeriod, - }, - Networks: []swarm.NetworkAttachmentConfig{ - { - Target: networkID, - Aliases: []string{dep.Key}, - }, - }, - RestartPolicy: &swarm.RestartPolicy{ - Condition: swarm.RestartPolicyConditionOnFailure, - MaxAttempts: &maxAttempts, - }, + exposedPort, portBindings, err := nat.ParsePortSpecs(dep.Ports) + containerConfig := &containertypes.Config{ + Image: dep.Image, + Labels: map[string]string{ + "mesg.engine": c.engineName, + "mesg.service": srv.Hash.String(), + "mesg.instance": instanceHash.String(), + "mesg.runner": runnerHash.String(), + "mesg.dependency": dep.Key, }, - EndpointSpec: &swarm.EndpointSpec{ - Ports: convertPorts(dep.Ports), + Env: dep.Env, + ExposedPorts: exposedPort, + } + hostConfig := &containertypes.HostConfig{ + PortBindings: portBindings, + Mounts: append(volumes, volumesFrom...), + RestartPolicy: containertypes.RestartPolicy{ + Name: "on-failure", + MaximumRetryCount: defaultMaximumRetryCount, }, - }) + } + if dep.Command != "" { + containerConfig.Cmd = append(containerConfig.Cmd, dep.Command) + } + containerConfig.Cmd = append(containerConfig.Cmd, dep.Args...) + + // create container, attach network, start it + if _, err := c.client.ContainerCreate(context.Background(), containerConfig, hostConfig, nil, depName); err != nil { + return err + } + if err := c.client.NetworkConnect(context.Background(), networkID, depName, &networktypes.EndpointSettings{ + Aliases: []string{dep.Key}, + }); err != nil { + return err + } + if err := c.client.ContainerStart(context.Background(), depName, types.ContainerStartOptions{}); err != nil { + return err + } } // Create configuration container config + exposedPort, portBindings, err := nat.ParsePortSpecs(srv.Configuration.Ports) + if err != nil { + return err + } volumes := convertVolumes(srv, srv.Configuration.Volumes, service.MainServiceKey) volumesFrom, err := convertVolumesFrom(srv, srv.Configuration.VolumesFrom) if err != nil { return err } - stopGracePeriod := defaultStopGracePeriod - maxAttempts := defaultMaxAttempts - specs = append(specs, swarm.ServiceSpec{ - Annotations: swarm.Annotations{ - Name: runnerName, - Labels: map[string]string{ - "mesg.engine": c.engineName, - "mesg.service": srv.Hash.String(), - "mesg.instance": instanceHash.String(), - "mesg.runner": runnerHash.String(), - "mesg.dependency": service.MainServiceKey, - "com.docker.stack.namespace": runnerName, - }, - }, - TaskTemplate: swarm.TaskSpec{ - ContainerSpec: &swarm.ContainerSpec{ - Image: imageTag + srv.Hash.String(), - Labels: map[string]string{ - "com.docker.stack.namespace": runnerName, - }, - Args: srv.Configuration.Args, - Command: strings.Fields(srv.Configuration.Command), - Env: xos.EnvMergeSlices(instanceEnv, []string{ - "MESG_SERVICE_HASH=" + srv.Hash.String(), - "MESG_INSTANCE_HASH=" + instanceHash.String(), - "MESG_RUNNER_HASH=" + runnerHash.String(), - "MESG_ENDPOINT=" + c.engineEndpoint, - "MESG_REGISTER_SIGNATURE=" + base64.StdEncoding.EncodeToString(registerPayload), - "MESG_ENV_HASH=" + instanceEnvHash.String(), - }), - Mounts: append(volumes, volumesFrom...), - StopGracePeriod: &stopGracePeriod, - }, - Networks: []swarm.NetworkAttachmentConfig{ - { - Target: networkID, - Aliases: []string{service.MainServiceKey}, - }, - { - Target: enginedNetworkID, - }, - }, - RestartPolicy: &swarm.RestartPolicy{ - Condition: swarm.RestartPolicyConditionOnFailure, - MaxAttempts: &maxAttempts, - }, + containerConfig := &containertypes.Config{ + Image: imageTag + srv.Hash.String(), + Labels: map[string]string{ + "mesg.engine": c.engineName, + "mesg.service": srv.Hash.String(), + "mesg.instance": instanceHash.String(), + "mesg.runner": runnerHash.String(), + "mesg.dependency": service.MainServiceKey, }, - EndpointSpec: &swarm.EndpointSpec{ - Ports: convertPorts(srv.Configuration.Ports), + Env: xos.EnvMergeSlices(instanceEnv, []string{ + "MESG_SERVICE_HASH=" + srv.Hash.String(), + "MESG_INSTANCE_HASH=" + instanceHash.String(), + "MESG_RUNNER_HASH=" + runnerHash.String(), + "MESG_ENDPOINT=" + c.engineEndpoint, + "MESG_REGISTER_SIGNATURE=" + base64.StdEncoding.EncodeToString(registerPayload), + "MESG_ENV_HASH=" + instanceEnvHash.String(), + }), + ExposedPorts: exposedPort, + } + hostConfig := &containertypes.HostConfig{ + PortBindings: portBindings, + Mounts: append(volumes, volumesFrom...), + RestartPolicy: containertypes.RestartPolicy{ + Name: "on-failure", + MaximumRetryCount: defaultMaximumRetryCount, }, - }) + } + if srv.Configuration.Command != "" { + containerConfig.Cmd = append(containerConfig.Cmd, srv.Configuration.Command) + } + containerConfig.Cmd = append(containerConfig.Cmd, srv.Configuration.Args...) - // Start - for _, spec := range specs { - if err := c.startService(spec); err != nil { - c.Stop(srv, runnerHash) - return err - } + // create container, attach networks, start it + if _, err := c.client.ContainerCreate(context.Background(), containerConfig, hostConfig, nil, runnerName); err != nil { + return err + } + if err := c.client.NetworkConnect(context.Background(), networkID, runnerName, &networktypes.EndpointSettings{ + Aliases: []string{service.MainServiceKey}, + }); err != nil { + return err + } + if err := c.client.NetworkConnect(context.Background(), enginedNetworkID, runnerName, &networktypes.EndpointSettings{}); err != nil { + return err + } + if err := c.client.ContainerStart(context.Background(), runnerName, types.ContainerStartOptions{}); err != nil { + return err } + errorOccurred = false return nil } @@ -300,7 +270,11 @@ func (c *Container) Stop(srv *service.Service, runnerHash hash.Hash) error { names = append(names, name) for _, name := range names { - if err := c.stopService(name); err != nil { + stopGracePeriod := defaultStopGracePeriod + if err := c.client.ContainerStop(context.Background(), name, &stopGracePeriod); err != nil { + errs.Append(err) + } + if err := c.client.ContainerRemove(context.Background(), name, types.ContainerRemoveOptions{}); err != nil { errs.Append(err) } } @@ -343,30 +317,11 @@ func (c *Container) Stop(srv *service.Service, runnerHash hash.Hash) error { // return errs.ErrorOrNil() // } -func convertPorts(dPorts []string) []swarm.PortConfig { - ports := make([]swarm.PortConfig, len(dPorts)) - for i, p := range dPorts { - split := strings.Split(p, ":") - from, _ := strconv.ParseUint(split[0], 10, 64) - to := from - if len(split) > 1 { - to, _ = strconv.ParseUint(split[1], 10, 64) - } - ports[i] = swarm.PortConfig{ - Protocol: swarm.PortConfigProtocolTCP, - PublishMode: swarm.PortConfigPublishModeIngress, - TargetPort: uint32(to), - PublishedPort: uint32(from), - } - } - return ports -} - -func convertVolumes(s *service.Service, dVolumes []string, key string) []mount.Mount { - volumes := make([]mount.Mount, 0) +func convertVolumes(s *service.Service, dVolumes []string, key string) []mounttypes.Mount { + volumes := make([]mounttypes.Mount, 0) for _, volume := range dVolumes { - volumes = append(volumes, mount.Mount{ - Type: mount.TypeVolume, + volumes = append(volumes, mounttypes.Mount{ + Type: mounttypes.TypeVolume, Source: volumeKey(s, key, volume), Target: volume, }) @@ -374,8 +329,8 @@ func convertVolumes(s *service.Service, dVolumes []string, key string) []mount.M return volumes } -func convertVolumesFrom(s *service.Service, dVolumesFrom []string) ([]mount.Mount, error) { - volumesFrom := make([]mount.Mount, 0) +func convertVolumesFrom(s *service.Service, dVolumesFrom []string) ([]mounttypes.Mount, error) { + volumesFrom := make([]mounttypes.Mount, 0) for _, depName := range dVolumesFrom { var depVolumes []string if depName == service.MainServiceKey { @@ -388,8 +343,8 @@ func convertVolumesFrom(s *service.Service, dVolumesFrom []string) ([]mount.Moun depVolumes = dep.Volumes } for _, volume := range depVolumes { - volumesFrom = append(volumesFrom, mount.Mount{ - Type: mount.TypeVolume, + volumesFrom = append(volumesFrom, mounttypes.Mount{ + Type: mounttypes.TypeVolume, Source: volumeKey(s, depName, volume), Target: volume, }) @@ -398,8 +353,6 @@ func convertVolumesFrom(s *service.Service, dVolumesFrom []string) ([]mount.Moun return volumesFrom, nil } -// volumeKey creates a key for service's volume based on the sid to make sure that the volume -// will stay the same for different versions of the service. func volumeKey(s *service.Service, dependency, volume string) string { return hash.Dump([]string{ s.Hash.String(), @@ -408,7 +361,6 @@ func volumeKey(s *service.Service, dependency, volume string) string { }).String() } -// createNetwork creates a Docker Network with a name. Retruns network id and error. func (c *Container) createNetwork(name string) (string, error) { network, err := c.client.NetworkInspect(context.Background(), name, types.NetworkInspectOptions{}) if err != nil && !client.IsErrNotFound(err) { @@ -419,206 +371,23 @@ func (c *Container) createNetwork(name string) (string, error) { } response, err := c.client.NetworkCreate(context.Background(), name, types.NetworkCreate{ CheckDuplicate: true, - Driver: "overlay", - Labels: map[string]string{ - "com.docker.stack.namespace": name, - }, + Driver: "bridge", }) return response.ID, err } -// deleteNetwork deletes a Docker Network. func (c *Container) deleteNetwork(name string) error { - for { - network, err := c.client.NetworkInspect(context.Background(), name, types.NetworkInspectOptions{}) - if client.IsErrNotFound(err) { - return nil - } - if err != nil { - return err - } - c.client.NetworkRemove(context.Background(), network.ID) - time.Sleep(pollingTime) - } -} - -// enginedNetworkID retrieve the docker network id of the engine network. -func (c *Container) enginedNetworkID() (string, error) { - network, err := c.client.NetworkInspect(context.Background(), c.engineNetwork, types.NetworkInspectOptions{}) - return network.ID, err -} - -// StartService starts a docker service. -func (c *Container) startService(spec swarm.ServiceSpec) error { - if status, _ := c.serviceStatus(spec.Name); status == RUNNING { - _, _, err := c.client.ServiceInspectWithRaw(context.Background(), spec.Name, types.ServiceInspectOptions{}) - return err - } - if _, err := c.client.ServiceCreate(context.Background(), spec, types.ServiceCreateOptions{}); err != nil { - return err - } - return c.waitForStatus(spec.Name, RUNNING) -} - -// StopService stops a docker service. -func (c *Container) stopService(name string) error { - status, err := c.serviceStatus(name) - if err != nil { - return err - } - if status == STOPPED { + network, err := c.client.NetworkInspect(context.Background(), name, types.NetworkInspectOptions{}) + if client.IsErrNotFound(err) { return nil } - service, _, err := c.client.ServiceInspectWithRaw(context.Background(), name, types.ServiceInspectOptions{}) - if err != nil && !client.IsErrNotFound(err) { - return err - } - stopGracePeriod := defaultStopGracePeriod - if service.Spec.TaskTemplate.ContainerSpec != nil && - service.Spec.TaskTemplate.ContainerSpec.StopGracePeriod != nil { - stopGracePeriod = *service.Spec.TaskTemplate.ContainerSpec.StopGracePeriod - } - - err = c.client.ServiceRemove(context.Background(), name) - if err != nil && !client.IsErrNotFound(err) { - return err - } - if err := c.deletePendingContainer(name, time.Now().Add(stopGracePeriod)); err != nil { - return err - } - return c.waitForStatus(name, STOPPED) -} - -// findContainer returns a docker container. -func (c *Container) findContainer(name string) (string, error) { - containers, err := c.client.ContainerList(context.Background(), types.ContainerListOptions{ - Filters: filters.NewArgs(filters.KeyValuePair{ - Key: "label", - Value: "com.docker.stack.namespace=" + name, - }), - Limit: 1, - }) - if err != nil { - return "", err - } - if len(containers) == 0 { - return "", errdefs.NotFound(fmt.Errorf("container in namespace %s not found", name)) - } - return containers[0].ID, nil -} - -// Status returns the status of the container based on the docker container and docker service. -// if any error occurs during the status check, status will be shown as UNKNOWN. -// otherwise the following rules will be applied to determine a status: -// - RUNNING: when the container is running in docker regardless of the status of the service. -// - STARTING: when the service is running but the container is not yet started. -// - STOPPED: when the container and the service is not running in docker. -func (c *Container) serviceStatus(name string) (Status, error) { - container, err := c.containerExists(name) - if err != nil { - return UNKNOWN, err - } - - service, err := c.serviceExists(name) - if err != nil { - return UNKNOWN, err - } - - for _, s := range statuses { - if s.container == container && s.service == service { - return s.status, nil - } - } - - // This should never be reached but it's better than a panic :) - return UNKNOWN, nil -} - -// containerExists checks if container with name can be found. -func (c *Container) containerExists(name string) (bool, error) { - _, err := c.findContainer(name) - if err != nil && !client.IsErrNotFound(err) { - return false, err - } - return !client.IsErrNotFound(err), nil -} - -// serviceExists checks if corresponding container for service namespace can be found. -func (c *Container) serviceExists(name string) (bool, error) { - _, _, err := c.client.ServiceInspectWithRaw(context.Background(), name, types.ServiceInspectOptions{}) - if err != nil && !client.IsErrNotFound(err) { - return false, err - } - return !client.IsErrNotFound(err), nil -} - -// tasksError returns the error of matching tasks. -func (c *Container) tasksError(name string) ([]string, error) { - tasks, err := c.client.TaskList(context.Background(), types.TaskListOptions{ - Filters: filters.NewArgs(filters.KeyValuePair{ - Key: "label", - Value: "com.docker.stack.namespace=" + name, - }), - }) - if err != nil { - return nil, err - } - - var errors []string - for _, task := range tasks { - if task.Status.Err != "" { - errors = append(errors, task.Status.Err) - } - } - return errors, nil -} - -// waitForStatus waits for the container to have the provided status. Returns error as soon as possible. -func (c *Container) waitForStatus(name string, status Status) error { - curstatus, err := c.serviceStatus(name) if err != nil { return err } - - for curstatus != status { - time.Sleep(pollingTime) - - tasksErrors, err := c.tasksError(name) - if err != nil { - return err - } - if len(tasksErrors) > 0 { - return errors.New(strings.Join(tasksErrors, ", ")) - } - - curstatus, err = c.serviceStatus(name) - if err != nil { - return err - } - } - - return nil + return c.client.NetworkRemove(context.Background(), network.ID) } -func (c *Container) deletePendingContainer(name string, maxGraceTime time.Time) error { - var ( - id string - err error - ) - for start := time.Now(); start.Before(maxGraceTime); time.Sleep(pollingTime) { - id, err = c.findContainer(name) - if client.IsErrNotFound(err) { - return nil - } - if err != nil { - return err - } - } - // Hack to force Docker to remove the containers. - // Sometime, the ServiceRemove function doesn't remove the associated containers, - // or too late and the associated networks cannot be removed. - // This hack for Docker to stop and then remove the container. - // See issue https://github.com/moby/moby/issues/32620 - c.client.ContainerRemove(context.Background(), id, types.ContainerRemoveOptions{Force: true}) - return nil +func (c *Container) enginedNetworkID() (string, error) { + network, err := c.client.NetworkInspect(context.Background(), c.engineNetwork, types.NetworkInspectOptions{}) + return network.ID, err } diff --git a/go.mod b/go.mod index e1134b598..18ae5627f 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/docker/cli v0.0.0-20191011045415-5d85cdacd257 github.com/docker/distribution v2.7.1+incompatible // indirect github.com/docker/docker v1.4.2-0.20191006173954-0abbb9e4ebf1 - github.com/docker/go-connections v0.4.0 // indirect + github.com/docker/go-connections v0.4.0 github.com/docker/go-units v0.4.0 // indirect github.com/go-bindata/go-bindata v0.0.0-20181025070752-41975c0ccc30 github.com/go-kit/kit v0.10.0 diff --git a/scripts/run-e2e.sh b/scripts/run-e2e.sh index 107085e16..044b29592 100755 --- a/scripts/run-e2e.sh +++ b/scripts/run-e2e.sh @@ -16,25 +16,24 @@ echo "run non existing test to detect compilation error quickly" go test -mod=readonly -v -count=1 ./e2e/... -run=__NONE__ function onexit { - docker service rm $ENGINE_NAME - docker wait $(docker ps -f label=com.docker.swarm.service.name=$ENGINE_NAME -q) 2> /dev/null - + docker stop $ENGINE_NAME docker network remove $NETWORK_NAME } trap onexit EXIT if [[ -z $(docker network list -f name="$NETWORK_NAME" -q) ]]; then - docker network create --driver overlay $NETWORK_NAME + docker network create $NETWORK_NAME fi -docker service create \ +docker run \ + -d \ + --rm \ --name $ENGINE_NAME \ -p 1317:1317 \ -p 50052:50052 \ -p 26657:26657 \ --network $NETWORK_NAME \ - --label com.docker.stack.namespace=$ENGINE_NAME \ mesg/engine:$1-dev echo "waiting lcd server to start" From 2ec4b30a626ac7bd349bf9bd6966a29d07939fa4 Mon Sep 17 00:00:00 2001 From: Nicolas Mahe Date: Fri, 8 May 2020 17:38:53 +0700 Subject: [PATCH 29/33] add missing step to pull image if it doesn't exist locally --- container/container.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/container/container.go b/container/container.go index d17e4861e..282941546 100644 --- a/container/container.go +++ b/container/container.go @@ -181,6 +181,16 @@ func (c *Container) Start(srv *service.Service, instanceHash, runnerHash, instan containerConfig.Cmd = append(containerConfig.Cmd, dep.Args...) // create container, attach network, start it + if _, _, err := c.client.ImageInspectWithRaw(context.Background(), dep.Image); err != nil { + r, err := c.client.ImagePull(context.Background(), dep.Image, types.ImagePullOptions{}) + if err != nil { + return err + } + // wait for docker to download the image + if _, err := ioutil.ReadAll(r); err != nil { + return err + } + } if _, err := c.client.ContainerCreate(context.Background(), containerConfig, hostConfig, nil, depName); err != nil { return err } From 29120cade8593dd1e1a60aa899ed3d7e15a0b878 Mon Sep 17 00:00:00 2001 From: Nicolas Mahe Date: Mon, 11 May 2020 14:55:43 +0700 Subject: [PATCH 30/33] make stopGracePeriod and maximumRetryCount configurable in container package --- container/container.go | 38 ++++++++++++++++++++------------------ e2e/api_test.go | 2 +- 2 files changed, 21 insertions(+), 19 deletions(-) diff --git a/container/container.go b/container/container.go index 282941546..515f685ca 100644 --- a/container/container.go +++ b/container/container.go @@ -28,11 +28,11 @@ import ( ) const ( - // defaultStopGracePeriod is the timeout value between stopping a container and killing it. - defaultStopGracePeriod = 10 * time.Second + // DefaultStopGracePeriod is the default timeout value between stopping a container and killing it. + DefaultStopGracePeriod = 10 * time.Second - // Number of time to restart the container if it crashes. - defaultMaximumRetryCount = 3 + // DefaultMaximumRetryCount is the default number of time to restart the container if it crashes. + DefaultMaximumRetryCount = 3 servicePrefix = "mesg_srv_" imageTag = "mesg:" @@ -40,16 +40,16 @@ const ( // Container starts and stops the MESG Service in Docker Container. type Container struct { - // client is a Docker client. - client client.CommonAPIClient - - engineEndpoint string - engineName string - engineNetwork string + client client.CommonAPIClient + engineEndpoint string + engineName string + engineNetwork string + stopGracePeriod time.Duration + maximumRetryCount int } // New initializes the Container struct by connecting creating the Docker client. -func New(engineName, engineAddress, engineNetwork string) (*Container, error) { +func New(engineName, engineAddress, engineNetwork string, maximumRetryCount int, stopGracePeriod time.Duration) (*Container, error) { client, err := client.NewClientWithOpts(client.FromEnv) if err != nil { return nil, err @@ -62,10 +62,12 @@ func New(engineName, engineAddress, engineNetwork string) (*Container, error) { } return &Container{ - client: client, - engineEndpoint: net.JoinHostPort(engineName, strconv.Itoa(port)), - engineName: engineName, - engineNetwork: engineNetwork, + client: client, + engineEndpoint: net.JoinHostPort(engineName, strconv.Itoa(port)), + engineName: engineName, + engineNetwork: engineNetwork, + stopGracePeriod: stopGracePeriod, + maximumRetryCount: maximumRetryCount, }, nil } @@ -172,7 +174,7 @@ func (c *Container) Start(srv *service.Service, instanceHash, runnerHash, instan Mounts: append(volumes, volumesFrom...), RestartPolicy: containertypes.RestartPolicy{ Name: "on-failure", - MaximumRetryCount: defaultMaximumRetryCount, + MaximumRetryCount: c.maximumRetryCount, }, } if dep.Command != "" { @@ -238,7 +240,7 @@ func (c *Container) Start(srv *service.Service, instanceHash, runnerHash, instan Mounts: append(volumes, volumesFrom...), RestartPolicy: containertypes.RestartPolicy{ Name: "on-failure", - MaximumRetryCount: defaultMaximumRetryCount, + MaximumRetryCount: c.maximumRetryCount, }, } if srv.Configuration.Command != "" { @@ -280,7 +282,7 @@ func (c *Container) Stop(srv *service.Service, runnerHash hash.Hash) error { names = append(names, name) for _, name := range names { - stopGracePeriod := defaultStopGracePeriod + stopGracePeriod := c.stopGracePeriod if err := c.client.ContainerStop(context.Background(), name, &stopGracePeriod); err != nil { errs.Append(err) } diff --git a/e2e/api_test.go b/e2e/api_test.go index 094731709..2be3cd6b4 100644 --- a/e2e/api_test.go +++ b/e2e/api_test.go @@ -92,7 +92,7 @@ func TestAPI(t *testing.T) { require.NoError(t, err) // init container - cont, err = container.New(orchestratorName, orchestratorEndpoint, orchestratorNetwork) + cont, err = container.New(orchestratorName, orchestratorEndpoint, orchestratorNetwork, 0, 5*time.Second) require.NoError(t, err) // init orchestrator gRPC client From abc499d5d8c4ec76a5cf4c816cbb0d0d21beb3a7 Mon Sep 17 00:00:00 2001 From: Nicolas Mahe Date: Mon, 11 May 2020 16:55:36 +0700 Subject: [PATCH 31/33] switch to tendermint logger add more log in orchestrator --- cmd/mesg-cli/orchestrator.go | 36 ++++------ go.mod | 2 +- orchestrator/orchestrator.go | 132 +++++++++++++++++++++++++---------- orchestrator/type.go | 22 ------ process/node.go | 29 ++++++++ server/grpc/logger.go | 22 ++++++ server/grpc/server.go | 14 ++-- 7 files changed, 172 insertions(+), 85 deletions(-) delete mode 100644 orchestrator/type.go create mode 100644 process/node.go create mode 100644 server/grpc/logger.go diff --git a/cmd/mesg-cli/orchestrator.go b/cmd/mesg-cli/orchestrator.go index 181de34be..c6a138fbf 100644 --- a/cmd/mesg-cli/orchestrator.go +++ b/cmd/mesg-cli/orchestrator.go @@ -48,19 +48,19 @@ func startOrchestratorCmd(cdc *codec.Codec) *cobra.Command { return fmt.Errorf("chain-id is required. use flag --chain-id or config file") } - logger := log.NewTMLogger(log.NewSyncWriter(os.Stdout)).With("module", "orchestrator") + logger := log.NewTMLogger(log.NewSyncWriter(os.Stdout)) client, err := cliCtx.GetNode() if err != nil { return err } // init rpc client - logger.Info("starting rpc client") + logger.Info("Starting rpc client") if err := client.Start(); err != nil { return err } defer func() { - logger.Info("stopping rpc client") + logger.Info("Stopping rpc client") if err := client.Stop(); err != nil { logger.Error(err.Error()) } @@ -81,39 +81,33 @@ func startOrchestratorCmd(cdc *codec.Codec) *cobra.Command { // init event publisher ep := publisher.New(rpc) - // init gRPC server. - logger.Info("starting grpc server") - server := grpc.New(rpc, ep, viper.GetStringSlice(flagAuthorizedPubKeys)) + // orchestrator + logger.Info("Starting orchestrator") + orch := orchestrator.New(rpc, ep, logger, viper.GetString(flagExecPrice)) defer func() { - logger.Info("stopping grpc server") - server.Close() + logger.Info("Stopping orchestrator") + orch.Stop() }() - go func() { - if err := server.Serve(viper.GetString(flagGrpcAddr)); err != nil { + if err := orch.Start(); err != nil { logger.Error(err.Error()) panic(err) } }() - // orchestrator - logger.Info("starting orchestrator") - orch := orchestrator.New(rpc, ep, viper.GetString(flagExecPrice)) + // init gRPC server. + logger.Info("Starting gRPC server") + server := grpc.New(rpc, ep, logger, viper.GetStringSlice(flagAuthorizedPubKeys)) defer func() { - logger.Info("stopping orchestrator") - orch.Stop() + logger.Info("Stopping gRPC server") + server.Close() }() go func() { - if err := orch.Start(); err != nil { + if err := server.Serve(viper.GetString(flagGrpcAddr)); err != nil { logger.Error(err.Error()) panic(err) } }() - go func() { - for err := range orch.ErrC { - logger.Error(err.Error()) - } - }() <-xsignal.WaitForInterrupt() diff --git a/go.mod b/go.mod index 18ae5627f..37347ee35 100644 --- a/go.mod +++ b/go.mod @@ -43,7 +43,7 @@ require ( github.com/prometheus/procfs v0.0.11 // indirect github.com/pseudomuto/protoc-gen-doc v1.3.1 github.com/rcrowley/go-metrics v0.0.0-20190706150252-9beb055b7962 // indirect - github.com/sirupsen/logrus v1.5.0 + github.com/sirupsen/logrus v1.5.0 // indirect github.com/spf13/afero v1.2.2 // indirect github.com/spf13/cobra v0.0.6 github.com/spf13/viper v1.6.3 diff --git a/orchestrator/orchestrator.go b/orchestrator/orchestrator.go index 0af227a51..3a7f4232e 100644 --- a/orchestrator/orchestrator.go +++ b/orchestrator/orchestrator.go @@ -2,7 +2,7 @@ package orchestrator import ( "context" - "errors" + "encoding/json" "fmt" "math/rand" @@ -20,20 +20,31 @@ import ( executionmodule "github.com/mesg-foundation/engine/x/execution" processmodule "github.com/mesg-foundation/engine/x/process" runnermodule "github.com/mesg-foundation/engine/x/runner" - "github.com/sirupsen/logrus" + tmlog "github.com/tendermint/tendermint/libs/log" ) func init() { xrand.SeedInit() } +// Orchestrator manages the executions based on the definition of the processes +type Orchestrator struct { + rpc *cosmos.RPC + ep *publisher.EventPublisher + eventStream *event.Listener + executionStream chan *execution.Execution + stopC chan bool + logger tmlog.Logger + execPrice string +} + // New creates a new Process instance -func New(rpc *cosmos.RPC, ep *publisher.EventPublisher, execPrice string) *Orchestrator { +func New(rpc *cosmos.RPC, ep *publisher.EventPublisher, logger tmlog.Logger, execPrice string) *Orchestrator { return &Orchestrator{ rpc: rpc, ep: ep, - ErrC: make(chan error), stopC: make(chan bool), + logger: logger.With("module", "orchestrator"), execPrice: execPrice, } } @@ -109,7 +120,7 @@ func (s *Orchestrator) findNodes(wf *process.Process, filter func(wf *process.Pr return wf.FindNodes(func(n *process.Process_Node) bool { res, err := filter(wf, n) if err != nil { - s.ErrC <- err + s.logger.With(keyvals(wf, n, nil, nil, nil)...).Error(err.Error()) } return res }) @@ -119,63 +130,88 @@ func (s *Orchestrator) execute(filter func(wf *process.Process, node *process.Pr var processes []*process.Process route := fmt.Sprintf("custom/%s/%s", processmodule.QuerierRoute, processmodule.QueryList) if err := s.rpc.QueryJSON(route, nil, &processes); err != nil { - s.ErrC <- err + s.logger.Error(err.Error()) return } for _, wf := range processes { for _, node := range s.findNodes(wf, filter) { if err := s.executeNode(wf, node, exec, event, data); err != nil { - s.ErrC <- err + s.logger.With(keyvals(wf, node, exec, event, data)...).Error(err.Error()) } } } } func (s *Orchestrator) executeNode(wf *process.Process, n *process.Process_Node, exec *execution.Execution, event *event.Event, data *types.Struct) error { - logrus.WithField("module", "orchestrator"). - WithField("node.key", n.Key). - WithField("type", fmt.Sprintf("%T", n)).Debug("process process") - if task := n.GetTask(); task != nil { + logData := keyvals(wf, n, exec, event, data) + // Process the node + switch x := n.Type.(type) { + case *process.Process_Node_Task_: // This returns directly because a task cannot process its children. // Children will be processed only when the execution is done and the dependencies are resolved - return s.processTask(n.Key, task, wf, exec, event, data) - } else if m := n.GetMap(); m != nil { + createdExecHash, err := s.processTask(n, x.Task, wf, exec, event, data) + if err != nil { + return err + } + s.logger.With(logData...).With("createdExecHash", createdExecHash.String()).Info("execution created") + return nil // stop workflow execution + case *process.Process_Node_Map_: var err error - data, err = s.processMap(n.Key, m.Outputs, wf, exec, data) + data, err = s.processMap(n.Key, x.Map.Outputs, wf, exec, data) if err != nil { return err } - } else if filter := n.GetFilter(); filter != nil { - if !s.filterMatch(filter, wf, n.Key, exec, data) { - return nil + log := s.logger.With(logData...) + if result, err := json.Marshal(data); err == nil { + log = log.With("output", string(result)) + } + log.Info("map executed") + case *process.Process_Node_Filter_: + log := s.logger.With(logData...) + if result, err := json.Marshal(x.Filter); err == nil { + log = log.With("filter", string(result)) + } + if !s.filterMatch(x.Filter, wf, n, exec, data) { + log.Info("filter is not matching") + return nil // stop workflow execution } + log.Info("filter is matching") + case *process.Process_Node_Result_, *process.Process_Node_Event_: + s.logger.With(logData...).Info("triggering process") + default: + return fmt.Errorf("unknown node type %q", n.TypeString()) } + + // Process the children for _, childrenID := range wf.ChildrenKeys(n.Key) { children, err := wf.FindNode(childrenID) if err != nil { // does not return an error to continue to process other tasks if needed - s.ErrC <- err + s.logger.With(logData...).Error(err.Error()) continue } if err := s.executeNode(wf, children, exec, event, data); err != nil { // does not return an error to continue to process other tasks if needed - s.ErrC <- err + s.logger.With(logData...).Error(err.Error()) + } } + return nil } // filterMatch returns true if the data match the current list of filters. -func (s *Orchestrator) filterMatch(f *process.Process_Node_Filter, wf *process.Process, nodeKey string, exec *execution.Execution, data *types.Struct) bool { +func (s *Orchestrator) filterMatch(f *process.Process_Node_Filter, wf *process.Process, n *process.Process_Node, exec *execution.Execution, data *types.Struct) bool { + logData := keyvals(wf, n, exec, nil, data) for _, condition := range f.Conditions { - resolvedData, err := s.resolveRef(wf, exec, nodeKey, data, condition.Ref) + resolvedData, err := s.resolveRef(wf, exec, n.Key, data, condition.Ref) if err != nil { - s.ErrC <- err + s.logger.With(logData...).Error(err.Error()) return false } match, err := condition.Match(resolvedData) if err != nil { - s.ErrC <- err + s.logger.With(logData...).Error(err.Error()) } if !match { return false @@ -236,7 +272,7 @@ func (s *Orchestrator) outputToValue(nodeKey string, output *process.Process_Nod case *process.Process_Node_Map_Output_Ref: return s.resolveRef(wf, exec, nodeKey, data, v.Ref) default: - return nil, errors.New("unknown output") + return nil, fmt.Errorf("unknown output") } } @@ -286,7 +322,7 @@ func (s *Orchestrator) resolveInput(wfHash hash.Hash, exec *execution.Execution, } // processTask create the request to execute the task. -func (s *Orchestrator) processTask(nodeKey string, task *process.Process_Node_Task, wf *process.Process, exec *execution.Execution, event *event.Event, data *types.Struct) error { +func (s *Orchestrator) processTask(node *process.Process_Node, task *process.Process_Node_Task, wf *process.Process, exec *execution.Execution, event *event.Event, data *types.Struct) (hash.Hash, error) { var eventHash, execHash hash.Hash if event != nil { eventHash = event.Hash @@ -297,7 +333,7 @@ func (s *Orchestrator) processTask(nodeKey string, task *process.Process_Node_Ta var runners []*runner.Runner route := fmt.Sprintf("custom/%s/%s", runnermodule.QuerierRoute, runnermodule.QueryList) if err := s.rpc.QueryJSON(route, nil, &runners); err != nil { - return err + return nil, err } executors := make([]*runner.Runner, 0) for _, run := range runners { @@ -306,31 +342,32 @@ func (s *Orchestrator) processTask(nodeKey string, task *process.Process_Node_Ta } } if len(executors) == 0 { - return fmt.Errorf("no runner is running instance %q", task.InstanceHash) + return nil, fmt.Errorf("no runner is running instance %q", task.InstanceHash) } executor := executors[rand.Intn(len(executors))] // create execution acc, err := s.rpc.GetAccount() if err != nil { - return err + return nil, err } msg := executionmodule.MsgCreate{ Signer: acc.GetAddress(), ProcessHash: wf.Hash, EventHash: eventHash, ParentHash: execHash, - NodeKey: nodeKey, + NodeKey: node.Key, TaskKey: task.TaskKey, Inputs: data, ExecutorHash: executor.Hash, Price: s.execPrice, Tags: nil, } - if _, err := s.rpc.BuildAndBroadcastMsg(msg); err != nil { - return err + res, err := s.rpc.BuildAndBroadcastMsg(msg) + if err != nil { + return nil, err } - return nil + return hash.DecodeFromBytes(res.Data) } // startExecutionStream returns execution that matches given hash. @@ -365,13 +402,13 @@ func (s *Orchestrator) startExecutionStream(ctx context.Context) error { attr := event.Events[attrKeyHash][index] hash, err := hash.Decode(attr) if err != nil { - s.ErrC <- err + s.logger.Error(err.Error()) continue } var exec *execution.Execution route := fmt.Sprintf("custom/%s/%s/%s", executionmodule.QuerierRoute, executionmodule.QueryGet, hash) if err := s.rpc.QueryJSON(route, nil, &exec); err != nil { - s.ErrC <- err + s.logger.Error(err.Error()) continue } s.executionStream <- exec @@ -381,8 +418,33 @@ func (s *Orchestrator) startExecutionStream(ctx context.Context) error { } } if err := s.rpc.Unsubscribe(context.Background(), subscriber, query); err != nil { - s.ErrC <- err + s.logger.Error(err.Error()) } }() return nil } + +func keyvals(proc *process.Process, node *process.Process_Node, parentExec *execution.Execution, event *event.Event, data *types.Struct) []interface{} { + keyvals := []interface{}{} + if proc != nil { + keyvals = append(keyvals, "processHash", proc.Hash.String()) + } + if node != nil { + keyvals = append(keyvals, + "nodeKey", node.Key, + "nodeType", node.TypeString(), + ) + } + if event != nil { + keyvals = append(keyvals, "eventHash", event.Hash.String()) + } + if parentExec != nil { + keyvals = append(keyvals, "parentHash", parentExec.Hash.String()) + } + if data != nil { + if result, err := json.Marshal(data); err == nil { + keyvals = append(keyvals, "data", string(result)) + } + } + return keyvals +} diff --git a/orchestrator/type.go b/orchestrator/type.go deleted file mode 100644 index b2034209a..000000000 --- a/orchestrator/type.go +++ /dev/null @@ -1,22 +0,0 @@ -package orchestrator - -import ( - "github.com/mesg-foundation/engine/cosmos" - "github.com/mesg-foundation/engine/event" - "github.com/mesg-foundation/engine/event/publisher" - "github.com/mesg-foundation/engine/execution" -) - -// Orchestrator manages the executions based on the definition of the processes -type Orchestrator struct { - rpc *cosmos.RPC - ep *publisher.EventPublisher - - eventStream *event.Listener - - executionStream chan *execution.Execution - ErrC chan error - stopC chan bool - - execPrice string -} diff --git a/process/node.go b/process/node.go new file mode 100644 index 000000000..84abcdcb7 --- /dev/null +++ b/process/node.go @@ -0,0 +1,29 @@ +package process + +// NodeType are the string constant corresponding to each type of node. +const ( + NodeTypeUnknown = "unknown" + NodeTypeResult = "result" + NodeTypeEvent = "event" + NodeTypeTask = "task" + NodeTypeMap = "map" + NodeTypeFilter = "filter" +) + +// TypeString returns the type of the node in string +func (node *Process_Node) TypeString() string { + switch node.Type.(type) { + case *Process_Node_Result_: + return NodeTypeResult + case *Process_Node_Event_: + return NodeTypeEvent + case *Process_Node_Task_: + return NodeTypeTask + case *Process_Node_Map_: + return NodeTypeMap + case *Process_Node_Filter_: + return NodeTypeFilter + default: + return NodeTypeUnknown + } +} diff --git a/server/grpc/logger.go b/server/grpc/logger.go new file mode 100644 index 000000000..3e094ad11 --- /dev/null +++ b/server/grpc/logger.go @@ -0,0 +1,22 @@ +package grpc + +import ( + tmlog "github.com/tendermint/tendermint/libs/log" +) + +func newTmLogger(logger tmlog.Logger, msg string) *tmLogger { + return &tmLogger{ + Logger: logger, + msg: msg, + } +} + +type tmLogger struct { + tmlog.Logger + msg string +} + +func (l tmLogger) Log(keyvals ...interface{}) error { + l.Info(l.msg, keyvals...) + return nil +} diff --git a/server/grpc/server.go b/server/grpc/server.go index 954349f60..15ff4b263 100644 --- a/server/grpc/server.go +++ b/server/grpc/server.go @@ -7,14 +7,14 @@ import ( "time" grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware" - grpc_logrus "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus" + grpc_kit "github.com/grpc-ecosystem/go-grpc-middleware/logging/kit" grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus" "github.com/mesg-foundation/engine/cosmos" "github.com/mesg-foundation/engine/event/publisher" "github.com/mesg-foundation/engine/ext/xvalidator" "github.com/mesg-foundation/engine/server/grpc/orchestrator" "github.com/mesg-foundation/engine/server/grpc/runner" - "github.com/sirupsen/logrus" + tmlog "github.com/tendermint/tendermint/libs/log" "google.golang.org/grpc" "google.golang.org/grpc/keepalive" "google.golang.org/grpc/reflection" @@ -25,14 +25,16 @@ type Server struct { instance *grpc.Server rpc *cosmos.RPC ep *publisher.EventPublisher + logger tmlog.Logger authorizedPubKeys []string } // New returns a new gRPC server. -func New(rpc *cosmos.RPC, ep *publisher.EventPublisher, authorizedPubKeys []string) *Server { +func New(rpc *cosmos.RPC, ep *publisher.EventPublisher, logger tmlog.Logger, authorizedPubKeys []string) *Server { return &Server{ rpc: rpc, ep: ep, + logger: logger.With("module", "grpc"), authorizedPubKeys: authorizedPubKeys, } } @@ -52,11 +54,11 @@ func (s *Server) Serve(address string) error { s.instance = grpc.NewServer( keepaliveOpt, grpc.StreamInterceptor(grpc_middleware.ChainStreamServer( - grpc_logrus.StreamServerInterceptor(logrus.StandardLogger().WithField("module", "grpc")), + grpc_kit.StreamServerInterceptor(newTmLogger(s.logger, "Served gRPC stream")), grpc_prometheus.StreamServerInterceptor, )), grpc.UnaryInterceptor(grpc_middleware.ChainUnaryServer( - grpc_logrus.UnaryServerInterceptor(logrus.StandardLogger().WithField("module", "grpc")), + grpc_kit.UnaryServerInterceptor(newTmLogger(s.logger, "Served gRPC response")), grpc_prometheus.UnaryServerInterceptor, validateInterceptor, )), @@ -65,7 +67,7 @@ func (s *Server) Serve(address string) error { return err } grpc_prometheus.Register(s.instance) - logrus.WithField("module", "grpc").Info("server listens on ", ln.Addr()) + s.logger.Info("Server listens on " + ln.Addr().String()) return s.instance.Serve(ln) } From a91b4b75d442439ed73c274736fa2c75eecadfe5 Mon Sep 17 00:00:00 2001 From: Nicolas Mahe Date: Mon, 11 May 2020 17:38:59 +0700 Subject: [PATCH 32/33] transform log to a more state transition approach --- orchestrator/orchestrator.go | 32 +++++++++++++------------------- 1 file changed, 13 insertions(+), 19 deletions(-) diff --git a/orchestrator/orchestrator.go b/orchestrator/orchestrator.go index 3a7f4232e..4ae6b4155 100644 --- a/orchestrator/orchestrator.go +++ b/orchestrator/orchestrator.go @@ -143,7 +143,7 @@ func (s *Orchestrator) execute(filter func(wf *process.Process, node *process.Pr } func (s *Orchestrator) executeNode(wf *process.Process, n *process.Process_Node, exec *execution.Execution, event *event.Event, data *types.Struct) error { - logData := keyvals(wf, n, exec, event, data) + log := s.logger.With(keyvals(wf, n, exec, event, data)...) // Process the node switch x := n.Type.(type) { case *process.Process_Node_Task_: @@ -153,7 +153,7 @@ func (s *Orchestrator) executeNode(wf *process.Process, n *process.Process_Node, if err != nil { return err } - s.logger.With(logData...).With("createdExecHash", createdExecHash.String()).Info("execution created") + log.With("createdExecHash", createdExecHash.String()).Info("execution created") return nil // stop workflow execution case *process.Process_Node_Map_: var err error @@ -161,38 +161,32 @@ func (s *Orchestrator) executeNode(wf *process.Process, n *process.Process_Node, if err != nil { return err } - log := s.logger.With(logData...) if result, err := json.Marshal(data); err == nil { log = log.With("output", string(result)) } - log.Info("map executed") case *process.Process_Node_Filter_: - log := s.logger.With(logData...) if result, err := json.Marshal(x.Filter); err == nil { log = log.With("filter", string(result)) } if !s.filterMatch(x.Filter, wf, n, exec, data) { - log.Info("filter is not matching") + log.Info("filter does not match data") return nil // stop workflow execution } - log.Info("filter is matching") - case *process.Process_Node_Result_, *process.Process_Node_Event_: - s.logger.With(logData...).Info("triggering process") - default: - return fmt.Errorf("unknown node type %q", n.TypeString()) } // Process the children for _, childrenID := range wf.ChildrenKeys(n.Key) { + log = log.With("to", childrenID) children, err := wf.FindNode(childrenID) if err != nil { // does not return an error to continue to process other tasks if needed - s.logger.With(logData...).Error(err.Error()) + log.Error(err.Error()) continue } + log.Info("executed process transition") if err := s.executeNode(wf, children, exec, event, data); err != nil { // does not return an error to continue to process other tasks if needed - s.logger.With(logData...).Error(err.Error()) + s.logger.With(keyvals(wf, children, exec, event, data)...).Error(err.Error()) } } @@ -202,16 +196,16 @@ func (s *Orchestrator) executeNode(wf *process.Process, n *process.Process_Node, // filterMatch returns true if the data match the current list of filters. func (s *Orchestrator) filterMatch(f *process.Process_Node_Filter, wf *process.Process, n *process.Process_Node, exec *execution.Execution, data *types.Struct) bool { - logData := keyvals(wf, n, exec, nil, data) + log := s.logger.With(keyvals(wf, n, exec, nil, data)...) for _, condition := range f.Conditions { resolvedData, err := s.resolveRef(wf, exec, n.Key, data, condition.Ref) if err != nil { - s.logger.With(logData...).Error(err.Error()) + log.Error(err.Error()) return false } match, err := condition.Match(resolvedData) if err != nil { - s.logger.With(logData...).Error(err.Error()) + log.Error(err.Error()) } if !match { return false @@ -431,8 +425,8 @@ func keyvals(proc *process.Process, node *process.Process_Node, parentExec *exec } if node != nil { keyvals = append(keyvals, - "nodeKey", node.Key, - "nodeType", node.TypeString(), + "from", node.Key, + "type", node.TypeString(), ) } if event != nil { @@ -443,7 +437,7 @@ func keyvals(proc *process.Process, node *process.Process_Node, parentExec *exec } if data != nil { if result, err := json.Marshal(data); err == nil { - keyvals = append(keyvals, "data", string(result)) + keyvals = append(keyvals, "input", string(result)) } } return keyvals From 1adf685edf5979d3e525ac28923c7484d3732043 Mon Sep 17 00:00:00 2001 From: Nicolas Mahe Date: Tue, 12 May 2020 14:37:57 +0700 Subject: [PATCH 33/33] add v0.24.0 to changelog --- CHANGELOG.md | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f2399ee22..47e0fb745 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,39 @@ # Changelog +## [v0.24.0](https://github.com/mesg-foundation/engine/releases/tag/v0.24.0) + +#### Breaking Changes + +- ([#1810](https://github.com/mesg-foundation/engine/pull/1810)) Switch to CLI based Engine. + +#### Added + +- ([#1798](https://github.com/mesg-foundation/engine/pull/1798)) Add tx commands for service, process, execution and runner modules. +- ([#1801](https://github.com/mesg-foundation/engine/pull/1801)) Add orchestrator command to CLI. +- ([#1806](https://github.com/mesg-foundation/engine/pull/1806)) Create pre-configured dev docker image. +- ([#1808](https://github.com/mesg-foundation/engine/pull/1808)) Add cli dockerfile. +- ([#1812](https://github.com/mesg-foundation/engine/pull/1812)) Add logs to orchestrator. + +#### Changed + +- ([#1802](https://github.com/mesg-foundation/engine/pull/1802)) Rename in cosmos lcd and rpc clients minGasPrices by simply gasPrices. +- ([#1803](https://github.com/mesg-foundation/engine/pull/1803)) Update readme. +- ([#1817](https://github.com/mesg-foundation/engine/pull/1817)) Use docker container in e2e tests. + +#### Fixed + +- ([#1760](https://github.com/mesg-foundation/engine/pull/1760)) Set grace period to 1m in dev script. +- ([#1809](https://github.com/mesg-foundation/engine/pull/1809)) Fix runner e2e that block the tests on error. + +#### Dependencies + +- ([#1745](https://github.com/mesg-foundation/engine/pull/1745)) Bump github.com/sirupsen/logrus from 1.4.2 to 1.5.0. +- ([#1777](https://github.com/mesg-foundation/engine/pull/1777)) Bump github.com/spf13/viper from 1.6.2 to 1.6.3. +- ([#1782](https://github.com/mesg-foundation/engine/pull/1782)) Bump github.com/golang/protobuf from 1.3.4 to 1.4.0. +- ([#1794](https://github.com/mesg-foundation/engine/pull/1794)) Bump google.golang.org/grpc from 1.28.0 to 1.29.1. +- ([#1800](https://github.com/mesg-foundation/engine/pull/1800)) Bump github.com/prometheus/client_golang from 1.5.0 to 1.6.0. +- ([#1811](https://github.com/mesg-foundation/engine/pull/1811)) Revert golang/protobuf to v1.3.5 and prometheus/client_golang to v1.5.1. + ## [v0.23.0](https://github.com/mesg-foundation/engine/releases/tag/v0.23.0) #### Added