Skip to content

Commit

Permalink
Add kafka backend (#2)
Browse files Browse the repository at this point in the history 
* add kafka as a new logging backend
* added required configuration definition to allow setting up a kafka logger
* updated the docker-compose to allow to run different backends
* update code to use the new logger
  • Loading branch information
@saryani
saryani authored Oct 8, 2024
1 parent e690601 commit b2fc5c6
Show file tree
Hide file tree
Showing 17 changed files with 1,294 additions and 33 deletions.
14 changes: 0 additions & 14 deletions .env.example
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,18 +11,4 @@ SESSION_KEY=sessionkey
OSCTRL_USER=admin
OSCTRL_PASS=Changeme123!
GOLANG_VERSION=1.23.0-bookworm

OSCTRL_TLS_LOGGER=db

LOG_KAFKA_BOOTSTRAP_SERVERS=my-kafka-bootstrap-server
LOG_KAFKA_SECURITY_PROTOCOL=SASL_SSL
LOG_KAFKA_SSLCA_LOCATION=path/to/kafka/ssl-ca
LOG_KAFKA_POLL_TIMEOUT=
LOG_KAFKA_CONNECTION_TIMEOUT=
LOG_KAFKA_CONSUMER_GROUP=
LOG_KAFKA_MESSAGE_MAX_BYTES=
LOG_KAFKA_COMMIT_INTERVAL=
LOG_KAFKA_TOPIC=kafka-topic-to-push-logs-to
LOG_KAFKA_SASL_MECHANISM=SCRAM-SHA-512
LOG_KAFKA_SASL_USERNAME=kafka-client-username
LOG_KAFKA_SASL_PASSWORD=kafka-client-password
342 changes: 342 additions & 0 deletions admin/handlers/go.sum 13-41-41-327.sum
View file

Large diffs are not rendered by default.

7 changes: 7 additions & 0 deletions admin/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -416,6 +416,13 @@ func init() {
EnvVars: []string{"DB_PASS"},
Destination: &dbConfigValues.Password,
},
&cli.StringFlag{
Name: "db-sslmode",
Value: "disable",
Usage: "SSL native support to encrypt the connection to the backend",
EnvVars: []string{"DB_SSLMODE"},
Destination: &dbConfigValues.SSLMode,
},
&cli.IntFlag{
Name: "db-max-idle-conns",
Value: 20,
Expand Down
89 changes: 89 additions & 0 deletions admin/sessions/go.sum 13-41-24-499.sum
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60=
<<<<<<< HEAD
github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
=======
github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
>>>>>>> main
github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg=
github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/gorilla/securecookie v1.1.2 h1:YCIWL56dvtr73r6715mJs5ZvhtnY73hBvEF8kXD8ePA=
github.com/gorilla/securecookie v1.1.2/go.mod h1:NfCASbcHqRSY+3a8tlWJwsQap2VX5pwzwo4h3eOamfo=
github.com/gorilla/sessions v1.4.0 h1:kpIYOp/oi6MG/p5PgxApU8srsSw9tuFbt46Lt7auzqQ=
github.com/gorilla/sessions v1.4.0/go.mod h1:FLWm50oby91+hl7p/wRxDth9bWSuk0qVL2emc7lT5ik=
github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8=
github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk=
github.com/jackc/pgconn v1.14.3 h1:bVoTr12EGANZz66nZPkMInAV/KHD2TxH9npjXXgiB3w=
github.com/jackc/pgconn v1.14.3/go.mod h1:RZbme4uasqzybK2RK5c65VsHxoyaml09lx3tXOcO/VM=
github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE=
github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8=
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
github.com/jackc/pgproto3/v2 v2.3.3 h1:1HLSx5H+tXR9pW3in3zaztoEwQYRC9SQaYUHjTSUOag=
github.com/jackc/pgproto3/v2 v2.3.3/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA=
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk=
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
github.com/jackc/pgtype v1.14.0 h1:y+xUdabmyMkJLyApYuPj38mW+aAIqCe5uuBB51rH3Vw=
github.com/jackc/pgtype v1.14.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4=
github.com/jackc/pgx/v4 v4.18.2 h1:xVpYkNR5pk5bMCZGfClbO962UIqVABcAGt7ha1s/FeU=
github.com/jackc/pgx/v4 v4.18.2/go.mod h1:Ey4Oru5tH5sB6tV7hDmfWFahwF15Eb7DNXlRKx2CkVw=
github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
<<<<<<< HEAD
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
=======
github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8=
github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss=
>>>>>>> main
github.com/segmentio/ksuid v1.0.4 h1:sBo2BdShXjmcugAMwjugoGUdUV0pcxY5mW4xKRn3v4c=
github.com/segmentio/ksuid v1.0.4/go.mod h1:/XUiZBD3kVx5SmUOl55voK5yeAbBNNIed+2O73XgrPE=
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw=
golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54=
<<<<<<< HEAD
=======
golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A=
golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM=
golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34=
golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc=
golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
>>>>>>> main
golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224=
golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gorm.io/driver/postgres v1.4.5 h1:mTeXTTtHAgnS9PgmhN2YeUbazYpLhUI1doLnw42XUZc=
gorm.io/driver/postgres v1.4.5/go.mod h1:GKNQYSJ14qvWkvPwXljMGehpKrhlDNsqYRr5HnYGncg=
gorm.io/gorm v1.25.11 h1:/Wfyg1B/je1hnDx3sMkX+gAlxrlZpn6X0BXRlwXlvHg=
gorm.io/gorm v1.25.11/go.mod h1:xh7N7RHfYlNc5EmcI/El95gXusucDrQnHXe0+CgWcLQ=
gorm.io/gorm v1.25.12 h1:I0u8i2hWQItBq1WfE0o2+WuL9+8L21K9e2HHSTE/0f8=
gorm.io/gorm v1.25.12/go.mod h1:xh7N7RHfYlNc5EmcI/El95gXusucDrQnHXe0+CgWcLQ=
7 changes: 7 additions & 0 deletions api/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -337,6 +337,13 @@ func init() {
EnvVars: []string{"DB_PASS"},
Destination: &dbConfigValues.Password,
},
&cli.StringFlag{
Name: "db-sslmode",
Value: "disable",
Usage: "SSL native support to encrypt the connection to the backend",
EnvVars: []string{"DB_SSLMODE"},
Destination: &dbConfigValues.SSLMode,
},
&cli.IntFlag{
Name: "db-max-idle-conns",
Value: 20,
Expand Down
5 changes: 3 additions & 2 deletions backend/backend.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ import (

const (
// DBString to format connection string to database for postgres
DBString = "host=%s port=%s dbname=%s user=%s password=%s sslmode=require"
DBString = "host=%s port=%s dbname=%s user=%s password=%s sslmode=%s"
// DBKey to identify the configuration JSON key
DBKey = "db"
)
Expand All @@ -31,6 +31,7 @@ type JSONConfigurationDB struct {
Name string `json:"name"`
Username string `json:"username"`
Password string `json:"password"`
SSLMode string `json:"sslmode"`
MaxIdleConns int `json:"maxIdleConns"`
MaxOpenConns int `json:"maxOpenConns"`
ConnMaxLifetime int `json:"connMaxLifetime"`
Expand All @@ -57,7 +58,7 @@ func LoadConfiguration(file, key string) (JSONConfigurationDB, error) {
// PrepareDSN to generate DB connection string
func PrepareDSN(config JSONConfigurationDB) string {
return fmt.Sprintf(
DBString, config.Host, config.Port, config.Name, config.Username, config.Password)
DBString, config.Host, config.Port, config.Name, config.Username, config.Password, config.SSLMode)
}

// GetDB to get PostgreSQL DB using GORM
Expand Down
1 change: 1 addition & 0 deletions deploy/config/db.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
"name": "_DB_NAME",
"username": "_DB_USERNAME",
"password": "_DB_PASSWORD",
"sslmode": "disable",
"maxIdleConns": 20,
"maxOpenConns": 100,
"connMaxLifetime": 30,
Expand Down
11 changes: 1 addition & 10 deletions docker-compose-dev.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,16 +53,7 @@ services:
- DB_USER=${POSTGRES_DB_USERNAME}
- DB_PASS=${POSTGRES_DB_PASSWORD}
#### Redis settings ####
- REDIS_HOST=osctrl-redis
#### Kafka settings ####
- LOG_KAFKA_BOOTSTRAP_SERVERS=${LOG_KAFKA_BOOTSTRAP_SERVERS}
- LOG_KAFKA_SECURITY_PROTOCOL=${LOG_KAFKA_SECURITY_PROTOCOL}
- LOG_KAFKA_SSLCA_LOCATION=${LOG_KAFKA_SSLCA_LOCATION}
- LOG_KAFKA_CONNECTION_TIMEOUT=${LOG_KAFKA_CONNECTION_TIMEOUT}
- LOG_KAFKA_TOPIC=${LOG_KAFKA_TOPIC}
- LOG_KAFKA_SASL_MECHANISM=${LOG_KAFKA_SASL_MECHANISM}
- LOG_KAFKA_SASL_USERNAME=${LOG_KAFKA_SASL_USERNAME}
- LOG_KAFKA_SASL_PASSWORD=${LOG_KAFKA_SASL_PASSWORD}
- REDIS_HOST=osctrl-redis
networks:
- osctrl-dev-backend
ports:
Expand Down
Loading

0 comments on commit b2fc5c6

Please sign in to comment.