Release v0.14.0

CHANGELOG.md

@@ -9,6 +9,58 @@
 #### Removed
 #### Experimental
 
+## [v0.14.0](https://github.com/mesg-foundation/engine/releases/tag/v0.14.0)
+
+#### Breaking Changes
+
+- ([#1258](https://github.com/mesg-foundation/engine/pull/1258)) Pure graph implementation for the orchestration system.
+- ([#1278](https://github.com/mesg-foundation/engine/pull/1278)) Rename workflow to process.
+- ([#1290](https://github.com/mesg-foundation/engine/pull/1290)) Use gogo protobuf.
+- ([#1292](https://github.com/mesg-foundation/engine/pull/1292)) Set nullabe to false for service configuration.
+- ([#1301](https://github.com/mesg-foundation/engine/pull/1301)) Update database path to ensure no issue on startup.
+
+#### Added
+
+- ([#1241](https://github.com/mesg-foundation/engine/pull/1241)) Process input resolution.
+
+#### Changed
+
+- ([#1254](https://github.com/mesg-foundation/engine/pull/1254)) Remove database ErrNotFound. Introduce Database Exist function.
+- ([#1260](https://github.com/mesg-foundation/engine/pull/1260)) Keep only one Service struct (the one generated from protobuf definition).
+- ([#1264](https://github.com/mesg-foundation/engine/pull/1264)) Refactor/remove custom service type no custom type hash.
+- ([#1269](https://github.com/mesg-foundation/engine/pull/1269)) Change service.proto package back to types.
+- ([#1270](https://github.com/mesg-foundation/engine/pull/1270)) Change proto hash type from string to bytes.
+- ([#1285](https://github.com/mesg-foundation/engine/pull/1285)) Bump github.com/gogo/protobuf from 1.2.1 to 1.3.0.
+- ([#1286](https://github.com/mesg-foundation/engine/pull/1286)) Fix marshal issue with process filter.
+- ([#1287](https://github.com/mesg-foundation/engine/pull/1287)) Update system services to work with latest gRPC API.
+- ([#1288](https://github.com/mesg-foundation/engine/pull/1288)) Fix that hash of info API were still in string instead of bytes.
+- ([#1291](https://github.com/mesg-foundation/engine/pull/1291)) Customize protobuf messages .
+- ([#1293](https://github.com/mesg-foundation/engine/pull/1293)) Replace instance struct with protobuf.
+- ([#1295](https://github.com/mesg-foundation/engine/pull/1295)) Remove old generated instance struct.
+- ([#1296](https://github.com/mesg-foundation/engine/pull/1296)) Remove and refactor rest protobuf types.
+- ([#1298](https://github.com/mesg-foundation/engine/pull/1298)) Replace cnf structhash with custom one.
+- ([#1310](https://github.com/mesg-foundation/engine/pull/1310)) Update gogo protobuf with fixed json and custom tags generation.
+- ([#1311](https://github.com/mesg-foundation/engine/pull/1311)) Use protobuf marshal/unmarshal in process database.
+- ([#1312](https://github.com/mesg-foundation/engine/pull/1312)) Use protobuf marshal/unmarshal in execution database.
+
+#### Experimental
+
+- ([#1243](https://github.com/mesg-foundation/engine/pull/1243)) Add store package.
+- ([#1244](https://github.com/mesg-foundation/engine/pull/1244)) Update service db to use store.
+- ([#1245](https://github.com/mesg-foundation/engine/pull/1245)) Bump github.com/cosmos/cosmos-sdk from 0.36.0 to 0.37.0.
+- ([#1248](https://github.com/mesg-foundation/engine/pull/1248)) Add Cosmos helpers.
+- ([#1249](https://github.com/mesg-foundation/engine/pull/1249)) SDK loads the App modules.
+- ([#1250](https://github.com/mesg-foundation/engine/pull/1250)) Transform service sdk to accept cosmos service step 1.
+- ([#1252](https://github.com/mesg-foundation/engine/pull/1252)) Add service sdk cosmos version step 2.
+- ([#1255](https://github.com/mesg-foundation/engine/pull/1255)) Create tendermint client.
+- ([#1257](https://github.com/mesg-foundation/engine/pull/1257)) Recover from panic in cosmos store.
+- ([#1268](https://github.com/mesg-foundation/engine/pull/1268)) Make cosmos store iterator works the same way as goleveldb iterator.
+- ([#1271](https://github.com/mesg-foundation/engine/pull/1271)) Bump github.com/tendermint/tendermint from 0.32.2 to 0.32.3.
+- ([#1284](https://github.com/mesg-foundation/engine/pull/1284)) Add test for both cosmos store and leveldb store.
+- ([#1297](https://github.com/mesg-foundation/engine/pull/1297)) Add explicit marshal and unmarshal function to service db.
+- ([#1299](https://github.com/mesg-foundation/engine/pull/1299)) Create a cosmos app factory struct.
+- ([#1300](https://github.com/mesg-foundation/engine/pull/1300)) Prepare cosmos client.
+
 ## [v0.13.0](https://github.com/mesg-foundation/engine/releases/tag/v0.13.0)
 
 #### Added

Dockerfile.tools

@@ -15,15 +15,28 @@ RUN apt-get update -y && \
     rm -rf /var/lib/apt/lists/*
 
 # protobuf(protoc) version.
-ARG protobuf=3.8.0
+ARG protobuf=3.9.1
+ARG gogoprotobuf=1.3.0
 
 # install protobuf(protoc).
 RUN wget -q https://github.com/protocolbuffers/protobuf/releases/download/v$protobuf/protoc-$protobuf-linux-x86_64.zip -O /tmp/protobuf.zip && \
     mkdir /tmp/protobuf && \
     unzip /tmp/protobuf.zip -d /tmp/protobuf && \
     mv /tmp/protobuf/bin/* /usr/local/bin/ && \
     mv /tmp/protobuf/include/* /usr/local/include/ && \
-    rm -rf /tmp/*
+    rm -rf /tmp/protobuf
+
+RUN cd /tmp/ && \
+    git clone https://github.com/krhubert/protobuf.git && \
+    cd protobuf && \
+    make install && \
+    mkdir -p /usr/local/include/gogo/protobuf/ && \
+    cp gogoproto/gogo.proto /usr/local/include/gogo/protobuf/
+
+# download gogo proto files
+RUN mkdir -p /usr/local/include/gogo/protobuf/ && \
+    wget -qO- https://github.com/gogo/protobuf/archive/v${gogoprotobuf}.tar.gz | tar -xzf - protobuf-${gogoprotobuf}/gogoproto/gogo.proto && \
+    mv protobuf-${gogoprotobuf}/gogoproto /usr/local/include/gogo/protobuf/
 
 RUN go install github.com/go-bindata/go-bindata/go-bindata
 RUN go install github.com/golang/protobuf/protoc-gen-go

config/config.go

@@ -19,10 +19,10 @@ import (
 const (
 	envPrefix = "mesg"
 
-	serviceDBVersion   = "v3"
-	executionDBVersion = "v2"
-	instanceDBVersion  = "v1"
-	workflowDBVersion  = "v1"
+	serviceDBVersion   = "v4"
+	executionDBVersion = "v3"
+	instanceDBVersion  = "v2"
+	processDBVersion   = "v2"
 )
 
 var (
@@ -49,7 +49,7 @@ type Config struct {
 		ServiceRelativePath   string
 		InstanceRelativePath  string
 		ExecutionRelativePath string
-		WorkflowRelativePath  string
+		ProcessRelativePath   string
 	}
 
 	Tendermint struct {
@@ -95,7 +95,7 @@ func New() (*Config, error) {
 	c.Database.ServiceRelativePath = filepath.Join("database", "services", serviceDBVersion)
 	c.Database.InstanceRelativePath = filepath.Join("database", "instance", instanceDBVersion)
 	c.Database.ExecutionRelativePath = filepath.Join("database", "executions", executionDBVersion)
-	c.Database.WorkflowRelativePath = filepath.Join("database", "workflows", workflowDBVersion)
+	c.Database.ProcessRelativePath = filepath.Join("database", "processes", processDBVersion)
 
 	c.Tendermint.Config = tmconfig.DefaultConfig()
 	c.Tendermint.Config.P2P.AddrBookStrict = false

core/main.go

@@ -6,83 +6,60 @@ import (
 	"strconv"
 	"sync"
 
+	"github.com/cosmos/cosmos-sdk/codec"
 	"github.com/mesg-foundation/engine/config"
 	"github.com/mesg-foundation/engine/container"
+	"github.com/mesg-foundation/engine/cosmos"
 	"github.com/mesg-foundation/engine/database"
+	"github.com/mesg-foundation/engine/database/store"
 	"github.com/mesg-foundation/engine/hash"
 	"github.com/mesg-foundation/engine/logger"
-	"github.com/mesg-foundation/engine/scheduler"
-	"github.com/mesg-foundation/engine/sdk"
+	"github.com/mesg-foundation/engine/orchestrator"
+	enginesdk "github.com/mesg-foundation/engine/sdk"
 	instancesdk "github.com/mesg-foundation/engine/sdk/instance"
 	servicesdk "github.com/mesg-foundation/engine/sdk/service"
 	"github.com/mesg-foundation/engine/server/grpc"
-	"github.com/mesg-foundation/engine/tendermint"
 	"github.com/mesg-foundation/engine/version"
 	"github.com/mesg-foundation/engine/x/xerrors"
 	"github.com/mesg-foundation/engine/x/xnet"
 	"github.com/mesg-foundation/engine/x/xos"
 	"github.com/mesg-foundation/engine/x/xsignal"
 	"github.com/sirupsen/logrus"
+	db "github.com/tendermint/tm-db"
 )
 
 var network = flag.Bool("experimental-network", false, "start the engine with the network")
 
-type dependencies struct {
-	cfg         *config.Config
-	serviceDB   database.ServiceDB
-	executionDB database.ExecutionDB
-	workflowDB  database.WorkflowDB
-	container   container.Container
-	sdk         *sdk.SDK
-}
-
-func initDependencies(cfg *config.Config) (*dependencies, error) {
+func initDatabases(cfg *config.Config) (*database.ServiceDB, *database.LevelDBInstanceDB, *database.LevelDBExecutionDB, *database.LevelDBProcessDB, error) {
 	// init services db.
-	serviceDB, err := database.NewServiceDB(filepath.Join(cfg.Path, cfg.Database.ServiceRelativePath))
+	serviceStore, err := store.NewLevelDBStore(filepath.Join(cfg.Path, cfg.Database.ServiceRelativePath))
 	if err != nil {
-		return nil, err
+		return nil, nil, nil, nil, err
 	}
+	serviceDB := database.NewServiceDB(serviceStore, codec.New())
 
 	// init instance db.
 	instanceDB, err := database.NewInstanceDB(filepath.Join(cfg.Path, cfg.Database.InstanceRelativePath))
 	if err != nil {
-		return nil, err
+		return nil, nil, nil, nil, err
 	}
 
 	// init execution db.
 	executionDB, err := database.NewExecutionDB(filepath.Join(cfg.Path, cfg.Database.ExecutionRelativePath))
 	if err != nil {
-		return nil, err
+		return nil, nil, nil, nil, err
 	}
 
-	// init workflow db.
-	workflowDB, err := database.NewWorkflowDB(filepath.Join(cfg.Path, cfg.Database.WorkflowRelativePath))
+	// init process db.
+	processDB, err := database.NewProcessDB(filepath.Join(cfg.Path, cfg.Database.ProcessRelativePath))
 	if err != nil {
-		return nil, err
+		return nil, nil, nil, nil, err
 	}
 
-	// init container.
-	c, err := container.New(cfg.Name)
-	if err != nil {
-		return nil, err
-	}
-
-	_, port, _ := xnet.SplitHostPort(cfg.Server.Address)
-
-	// init sdk.
-	sdk := sdk.New(c, serviceDB, instanceDB, executionDB, workflowDB, cfg.Name, strconv.Itoa(port))
-
-	return &dependencies{
-		cfg:         cfg,
-		container:   c,
-		serviceDB:   serviceDB,
-		executionDB: executionDB,
-		workflowDB:  workflowDB,
-		sdk:         sdk,
-	}, nil
+	return serviceDB, instanceDB, executionDB, processDB, nil
 }
 
-func deployCoreServices(cfg *config.Config, sdk *sdk.SDK) error {
+func deployCoreServices(cfg *config.Config, sdk *enginesdk.SDK) error {
 	for _, serviceConfig := range cfg.SystemServices {
 		logrus.WithField("module", "main").Infof("Deploying service %q", serviceConfig.Definition.Sid)
 		srv, err := sdk.Service.Create(serviceConfig.Definition)
@@ -97,7 +74,7 @@ func deployCoreServices(cfg *config.Config, sdk *sdk.SDK) error {
 				return err
 			}
 		}
-		logrus.WithField("module", "main").Infof("Service %q deployed with hash %q", srv.Sid, srv.Hash)
+		logrus.WithField("module", "main").Infof("Service %q deployed with hash %q", srv.Sid, srv.Hash.String())
 		instance, err := sdk.Instance.Create(srv.Hash, xos.EnvMapToSlice(serviceConfig.Env))
 		if err != nil {
 			existsError, ok := err.(*instancesdk.AlreadyExistsError)
@@ -116,7 +93,7 @@ func deployCoreServices(cfg *config.Config, sdk *sdk.SDK) error {
 	return nil
 }
 
-func stopRunningServices(sdk *sdk.SDK) error {
+func stopRunningServices(sdk *enginesdk.SDK) error {
 	instances, err := sdk.Instance.List(&instancesdk.Filter{})
 	if err != nil {
 		return err
@@ -152,33 +129,75 @@ func main() {
 		logrus.Fatalln(err)
 	}
 
-	dep, err := initDependencies(cfg)
+	// init logger.
+	logger.Init(cfg.Log.Format, cfg.Log.Level, cfg.Log.ForceColors)
+
+	// init databases
+	serviceDB, instanceDB, executionDB, processDB, err := initDatabases(cfg)
+	if err != nil {
+		logrus.WithField("module", "main").Fatalln(err)
+	}
+
+	// init container.
+	c, err := container.New(cfg.Name)
 	if err != nil {
 		logrus.WithField("module", "main").Fatalln(err)
 	}
 
-	// init logger.
-	logger.Init(cfg.Log.Format, cfg.Log.Level, cfg.Log.ForceColors)
+	_, port, _ := xnet.SplitHostPort(cfg.Server.Address)
 
+	var sdk *enginesdk.SDK
 	if *network {
-		// create tendermint node
-		node, err := tendermint.NewNode(cfg.Tendermint.Config, &cfg.Cosmos)
+		// init app factory
+		db, err := db.NewGoLevelDB("app", cfg.Cosmos.Path)
 		if err != nil {
 			logrus.WithField("module", "main").Fatalln(err)
 		}
+		appFactory := cosmos.NewAppFactory(logger.TendermintLogger(), db)
+
+		// register the backend modules to the app factory.
+		enginesdk.NewBackend(appFactory, c)
+
+		// init cosmos app
+		app, err := cosmos.NewApp(appFactory)
+		if err != nil {
+			logrus.WithField("module", "main").Fatalln(err)
+		}
+
+		// init key manager
+		kb, err := cosmos.NewKeybase(cfg.Cosmos.Path)
+		if err != nil {
+			logrus.WithField("module", "main").Fatalln(err)
+		}
+
+		// create cosmos node
+		node, err := cosmos.NewNode(app, kb, cfg.Tendermint.Config, &cfg.Cosmos)
+		if err != nil {
+			logrus.WithField("module", "main").Fatalln(err)
+		}
+
+		// init sdk
+		sdk, err = enginesdk.New(c, instanceDB, executionDB, processDB, cfg.Name, strconv.Itoa(port))
+		if err != nil {
+			logrus.WithField("module", "main").Fatalln(err)
+		}
+
+		// start tendermint node
 		logrus.WithField("module", "main").WithField("seeds", cfg.Tendermint.P2P.Seeds).Info("starting tendermint node")
 		if err := node.Start(); err != nil {
 			logrus.WithField("module", "main").Fatalln(err)
 		}
-	}
+	} else {
+		sdk = enginesdk.NewDeprecated(c, serviceDB, instanceDB, executionDB, processDB, cfg.Name, strconv.Itoa(port))
 
-	// init system services.
-	if err := deployCoreServices(dep.cfg, dep.sdk); err != nil {
-		logrus.WithField("module", "main").Fatalln(err)
+		// init system services.
+		if err := deployCoreServices(cfg, sdk); err != nil {
+			logrus.WithField("module", "main").Fatalln(err)
+		}
 	}
 
 	// init gRPC server.
-	server := grpc.New(dep.sdk)
+	server := grpc.New(sdk)
 
 	logrus.WithField("module", "main").Infof("starting MESG Engine version %s", version.Version)
 
@@ -188,8 +207,8 @@ func main() {
 		}
 	}()
 
-	logrus.WithField("module", "main").Info("starting workflow engine")
-	s := scheduler.New(dep.sdk.Event, dep.sdk.Execution, dep.sdk.Workflow)
+	logrus.WithField("module", "main").Info("starting process engine")
+	s := orchestrator.New(sdk.Event, sdk.Execution, sdk.Process)
 	go func() {
 		if err := s.Start(); err != nil {
 			logrus.WithField("module", "main").Fatalln(err)
@@ -202,7 +221,7 @@ func main() {
 	}()
 
 	<-xsignal.WaitForInterrupt()
-	if err := stopRunningServices(dep.sdk); err != nil {
+	if err := stopRunningServices(sdk); err != nil {
 		logrus.WithField("module", "main").Fatalln(err)
 	}
 	server.Close()