Merge pull request #552 from xhdnoah/master

feat: otel tracing

.github/dependabot.yml

@@ -21,15 +21,3 @@ updates:
     - 1.10.1
     - 1.10.2
     - 1.10.3
-  - dependency-name: go.elastic.co/apm
-    versions:
-    - 1.10.0
-  - dependency-name: go.elastic.co/apm/module/apmsql
-    versions:
-    - 1.10.0
-  - dependency-name: go.elastic.co/apm/module/apmgrpc
-    versions:
-    - 1.10.0
-  - dependency-name: go.elastic.co/apm/module/apmgoredis
-    versions:
-    - 1.10.0

.github/workflows/ci.yml

@@ -25,7 +25,7 @@ jobs:
     runs-on: "${{ matrix.os }}"
     strategy:
       matrix:
-        go-version: ['1.18', '1.19']
+        go-version: ['1.21']
         os: [ubuntu-latest]
 
     services:

app.go

@@ -2,16 +2,17 @@ package gobay
 
 import (
 	"bytes"
+	"context"
 	"errors"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strings"
 	"sync"
 
-	"github.com/spf13/viper"
 	"github.com/hashicorp/go-multierror"
+	"github.com/shanbay/gobay/observability"
+	"github.com/spf13/viper"
 )
 
 // A Key represents a key for a Extension.
@@ -34,6 +35,7 @@ type Application struct {
 	initialized bool
 	closed      bool
 	mu          sync.Mutex
+	shutdown    func(context.Context) error
 }
 
 // Get the extension at the specified key, return nil when the component doesn't exist
@@ -72,6 +74,7 @@ func (d *Application) Init() error {
 	if err := d.initConfig(); err != nil {
 		return err
 	}
+	d.setup()
 	if err := d.initExtensions(); err != nil {
 		return err
 	}
@@ -81,7 +84,7 @@ func (d *Application) Init() error {
 
 func (d *Application) initConfig() error {
 	configfile := filepath.Join(d.rootPath, "config.yaml")
-	originConfig, err := ioutil.ReadFile(configfile)
+	originConfig, err := os.ReadFile(configfile)
 	if err != nil {
 		return err
 	}
@@ -101,12 +104,17 @@ func (d *Application) initConfig() error {
 	config.SetDefault("grpc_listen_port", 6000)
 	config.SetDefault("openapi_listen_host", "localhost")
 	config.SetDefault("openapi_listen_port", 3000)
+	config.SetDefault("otel_service_address", "otel-collector.guardian.svc.cluster.local:4317")
 
 	d.config = config
 
 	return nil
 }
 
+func (d *Application) setup() {
+	d.shutdown = observability.InitOtel(d.config)
+}
+
 func (d *Application) initExtensions() error {
 	var allerr error
 	for key, ext := range d.extensions {
@@ -129,6 +137,12 @@ func (d *Application) Close() error {
 	if err := d.closeExtensions(); err != nil {
 		return err
 	}
+	if d.shutdown != nil {
+		err := d.shutdown(context.Background())
+		if err != nil {
+			return err
+		}
+	}
 	d.closed = true
 	return nil
 }

cmd/gobay/main.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"bytes"
-	"io/ioutil"
 	"log"
 	"os"
 	"path"
@@ -26,12 +25,12 @@ type _projDir struct {
 }
 
 type _projConfig struct {
-	Url            string
-	Name           string
-	SkipSentry     bool
-	SkipAsyncTask  bool
-	SkipCache      bool
-	SkipElasticApm bool
+	Url           string
+	Name          string
+	SkipSentry    bool
+	SkipAsyncTask bool
+	SkipCache     bool
+	SkipOtel      bool
 }
 
 var (
@@ -78,7 +77,7 @@ func main() {
 	}
 	cmdNew.Flags().StringVar(&projConfig.Name, "name", "", "specific project name")
 	cmdNew.Flags().BoolVar(&projConfig.SkipSentry, "skip-sentry", false, "skip sentry")
-	cmdNew.Flags().BoolVar(&projConfig.SkipElasticApm, "skip-elasticapm", false, "skip elastic APM")
+	cmdNew.Flags().BoolVar(&projConfig.SkipOtel, "skip-otel", false, "skip otel")
 	cmdNew.Flags().BoolVar(&projConfig.SkipCache, "skip-cache", false, "skip cache")
 	cmdNew.Flags().BoolVar(&projConfig.SkipAsyncTask, "skip-asynctask", false, "skip asynctask")
 
@@ -174,7 +173,7 @@ func renderTemplates() {
 		if b.Len() <= 1 {
 			continue
 		}
-		if err := ioutil.WriteFile(f.dstPath, b.Bytes(), f.mode); err != nil {
+		if err := os.WriteFile(f.dstPath, b.Bytes(), f.mode); err != nil {
 			log.Fatalln(err)
 		}
 	}
@@ -195,7 +194,7 @@ func copyTmplFiles() {
 		if _, err := file.Read(b); err != nil {
 			panic(err)
 		}
-		check(ioutil.WriteFile(targetPath, b, FILEMODE))
+		check(os.WriteFile(targetPath, b, FILEMODE))
 	}
 }
 

cmd/gobay/templates/app/extensions.go.tmpl

@@ -13,18 +13,11 @@ import (
 	"github.com/shanbay/gobay/extensions/cachext"
 	_ "github.com/shanbay/gobay/extensions/cachext/backend/redis"
 	// "github.com/shanbay/gobay/extensions/entext"
-{{- if not $.SkipElasticApm }}
-	"github.com/shanbay/gobay/extensions/esapmext"
-{{- end }}
 	"github.com/shanbay/gobay/extensions/redisext"
 {{- if not $.SkipSentry }}
 	"github.com/shanbay/gobay/extensions/sentryext"
 {{- end }}
 	"github.com/shanbay/gobay/extensions/seqgenext"
-{{- if not $.SkipElasticApm }}
-	"go.elastic.co/apm"
-	_ "go.elastic.co/apm/module/apmsql/mysql"
-{{- end }}
 )
 
 // Extensions defined Extensions to be used by init app
@@ -36,9 +29,6 @@ func Extensions() map[gobay.Key]gobay.Extension {
 		"sentry": &sentryext.SentryExt{NS: "sentry_"},
 {{- end }}
 		"cache": &cachext.CacheExt{NS: "cache_"},
-{{- if not $.SkipElasticApm }}
-		"apm":   &esapmext.EsApmExt{},
-{{- end }}
 		// "entext": &entext.EntExt{
 		// 	NS: "db_",
 		// 	NewClient: func(opt interface{}) entext.Client {
@@ -61,9 +51,6 @@ var (
 	Sentry    *sentryext.SentryExt
 {{- end }}
 	Cache     *cachext.CacheExt
-{{- if not $.SkipElasticApm }}
-	ApmTracer *apm.Tracer
-{{- end }}
 {{- if not $.SkipAsyncTask }}
 	AsyncTask *asynctaskext.AsyncTaskExt
 {{- end }}
@@ -80,9 +67,6 @@ func InitExts(app *gobay.Application) {
 	Sentry = app.Get("sentry").Object().(*sentryext.SentryExt)
 {{- end }}
 	Cache = app.Get("cache").Object().(*cachext.CacheExt)
-{{- if not $.SkipElasticApm }}
-	ApmTracer = app.Get("apm").Object().(*apm.Tracer)
-{{- end }}
 {{- if not $.SkipAsyncTask }}
 	AsyncTask = app.Get("asyncTask").Object().(*asynctaskext.AsyncTaskExt)
 {{- end }}

cmd/gobay/templates/app/grpc/server.go.tmpl

@@ -16,9 +16,6 @@ import (
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 {{- if not $.SkipSentry }}
 	sentrygrpcmw "github.com/shanbay/gobay/extensions/sentryext/grpc"
-{{- end }}
-{{- if not $.SkipElasticApm }}
-	"go.elastic.co/apm/module/apmgrpc"
 {{- end }}
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/health/grpc_health_v1"
@@ -45,11 +42,6 @@ func Serve(app *gobay.Application) error {
 {{- end }}
 		grpc_prometheus.UnaryServerInterceptor,
 	}
-{{- if not $.SkipElasticApm }}
-	if config.GetBool("elastic_apm_enable") {
-		unaryInterceptors = append(unaryInterceptors, apmgrpc.NewUnaryServerInterceptor(apmgrpc.WithTracer(myapp.ApmTracer)))
-	}
-{{- end }}
 	streamInterceptors := []grpc.StreamServerInterceptor{
 		grpc_recovery.StreamServerInterceptor(),
 		grpc_prometheus.StreamServerInterceptor,

cmd/gobay/templates/app/oapi/server.go.tmpl

@@ -24,16 +24,13 @@ import (
 	"github.com/shanbay/gobay"
 	"github.com/shanbay/gobay/echo/swagger"
 	"github.com/shanbay/gobay/extensions/sentryext/custom_logger"
-{{- if not $.SkipElasticApm }}
-	"go.elastic.co/apm/module/apmechov4"
-{{- end }}
 )
 
 // Serve serve OpenAPI http server
 func Serve(
 	app *gobay.Application,
 	preInitFunc func(a *gobay.Application),
-	preStartFunc func(e *echo.Echo, a *gobay.Application, enableAPM bool, enableDoc bool),
+	preStartFunc func(e *echo.Echo, a *gobay.Application, enableDoc bool),
 	mocked bool,
 ) error {
 	// Initialize the project before loading the configuration
@@ -46,11 +43,6 @@ func Serve(
 	e := echo.New()
 	host := config.GetString("oapi_listen_host")
 	port := config.GetInt("oapi_listen_port")
-{{- if not $.SkipElasticApm }}
-	enableAPM := config.GetBool("elastic_apm_enable")
-{{- else }}
-	enableAPM := false
-{{- end }}
 
 	// config http keep-alive
 	var keepAlive time.Duration
@@ -66,7 +58,7 @@ func Serve(
 	e.Server.IdleTimeout = keepAlive
 
 	// Do something before start echo server
-	preStartFunc(e, app, enableAPM, true)
+	preStartFunc(e, app, true)
 
 	// 启动 prometheus handler
 	promSrv := http.Server{Addr: fmt.Sprintf(":%d", config.GetInt("oapi_prom_listen_port"))}
@@ -100,7 +92,7 @@ func Serve(
 	return nil
 }
 
-func configureMiddlewares(e *echo.Echo, impls *{{ toLowerCamel $.Name }}Server, enableAPM, enableDoc bool) {
+func configureMiddlewares(e *echo.Echo, impls *{{ toLowerCamel $.Name }}Server, enableDoc bool) {
 	oapi.RegisterHandlersWithBaseURL(e, impls, "/{{ $.Name }}")
 
 	// swagger
@@ -117,21 +109,12 @@ func configureMiddlewares(e *echo.Echo, impls *{{ toLowerCamel $.Name }}Server,
 		}
 	}
 
-	// APM or recover
-	if enableAPM {
-{{- if not $.SkipElasticApm }}
-		// APM middleware 会做 recover，但是不打印错误
-		// APM 会把 transaction 放在 echo.Context.Request().Context() 里
-		e.Use(apmechov4.Middleware(apmechov4.WithTracer(myapp.ApmTracer)))
-{{- end }}
-	} else {
-		e.Use(middleware.RecoverWithConfig(middleware.RecoverConfig{
-			StackSize: 1 << 10, // 1 KB
-			LogLevel:  echolog.ERROR,
-		}))
-	}
+	e.Use(middleware.RecoverWithConfig(middleware.RecoverConfig{
+		StackSize: 1 << 10, // 1 KB
+		LogLevel:  echolog.ERROR,
+	}))
 
-	// 错误处理流程（靠左的更外层）: recover (apm) => sentry => custom error handler
+	// 错误处理流程（靠左的更外层）: recover => sentry => custom error handler
 	e.Use(sentryecho.New(sentryecho.Options{Repanic: true}))
 
 	// Validator
@@ -167,7 +150,7 @@ func PreInitFunc(app *gobay.Application) {
 	myapp.InitExts(app)
 }
 
-func PreStartFunc(e *echo.Echo, app *gobay.Application, enableAPM, enableDoc bool) {
+func PreStartFunc(e *echo.Echo, app *gobay.Application, enableDoc bool) {
 	impls := &{{ toLowerCamel $.Name }}Server{app: app}
-	configureMiddlewares(e, impls, enableAPM, enableDoc)
+	configureMiddlewares(e, impls, enableDoc)
 }

cmd/gobay/templates/config.yaml.tmpl

@@ -22,14 +22,6 @@ defaults: &defaults
   sentry_environment: 'development'
 {{- end }}
 
-{{- if not $.SkipElasticApm }}
-  elastic_apm_service_name: '{{ $.Name }}-rpc-go'
-  elastic_apm_service_version: '1'
-  elastic_apm_server_url: ''
-  elastic_apm_transaction_sample_rate: 0.01
-  elastic_apm_enable: false
-{{- end }}
-
   oapi_listen_host: "0.0.0.0"
   oapi_listen_port: 5000
   oapi_prom_listen_port: 9000
@@ -55,9 +47,6 @@ testing:
 {{- end }}
 integration: &integration
   <<: *defaults
-{{- if not $.SkipElasticApm }}
-  elastic_apm_enable: true
-{{- end }}
   redis_addr: ${REDIS_ADDR}
   redis_password: ${REDIS_PASSWORD}
 {{- if not $.SkipSentry }}
@@ -71,11 +60,5 @@ integration: &integration
   db_url: ${DB_URL}
 staging: &staging
   <<: *integration
-{{- if not $.SkipElasticApm }}
-  elastic_apm_enable: false
-{{- end }}
 production:
   <<: *staging
-{{- if not $.SkipElasticApm }}
-  elastic_apm_enable: true
-{{- end }}

docs/CHANGELOG.md

@@ -1,9 +1,9 @@
-# [Unreleased]
-
-## Added:
+# 1.2.0
 
+- 升级 go1.21
 - 添加用于执行定时任务的 CronJobExt
-- 添加 golang 1.18 测试
+- 新增 go-redis/v9 作为可选升级项
+- 将 APM 替换成 OpenTelemetry 实现
 
 # 1.0.0 (2022-01-05)
 

docs/ext_database_cn.md

@@ -22,7 +22,6 @@ import (
   _ "github.com/go-sql-driver/mysql"
   "github.com/shanbay/gobay"
   "github.com/shanbay/gobay/extensions/entext"
-  _ "go.elastic.co/apm/module/apmsql/mysql"
 )
 
 func Extensions() map[gobay.Key]gobay.Extension {

docs/ext_grpc_client_cn.md

@@ -36,7 +36,6 @@ import (
   _ "github.com/go-sql-driver/mysql"
   "github.com/shanbay/gobay"
   "github.com/shanbay/gobay/extensions/entext"
-  _ "go.elastic.co/apm/module/apmsql/mysql"
 )
 
 func Extensions() map[gobay.Key]gobay.Extension {

docs/server_openapi_cn.md

@@ -92,7 +92,7 @@ make genswagger
 
 ```go
 // i.e.
-func configureAPI(s *restapi.Server, api *operations.HelloworldProjectAPI, impls *helloworldProjectServer, enableApm bool) {
+func configureAPI(s *restapi.Server, api *operations.HelloworldProjectAPI, impls *helloworldProjectServer) {
   // ...
   api.HealthHealthCheckHandler = impls.healthCheckHealthHandler()
   // ...

docs/structure.md

@@ -8,7 +8,7 @@
 │   │   ├── handlers.go
 │   │   └── server.go
 │   ├── constant.go # define constants here
-│   ├── extensions.go # define and prepare extensions (redis/mysql/grpc/cache/sentry/elasicApm/etc.)
+│   ├── extensions.go # define and prepare extensions (redis/mysql/grpc/cache/sentry/etc.)
 │   ├── grpc # GRPC service controllers
 │   │   ├── handlers.go
 │   │   └── server.go