From 66b790c8a39972ce7d834f3b5a42b629f029c8de Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Wed, 21 Aug 2024 08:37:07 +0200 Subject: [PATCH] api: add report timescale db --- api/configuration/configuration.go | 9 ++++++ api/go.mod | 12 +++++--- api/go.sum | 14 +++++++++ api/main.go | 7 +++++ api/middleware/middleware.go | 31 +++++++++++++++++++ api/storage/report_schema.sql | 14 +++++++++ api/storage/storage.go | 49 ++++++++++++++++++++++++++++++ 7 files changed, 132 insertions(+), 4 deletions(-) create mode 100644 api/storage/report_schema.sql diff --git a/api/configuration/configuration.go b/api/configuration/configuration.go index bb8f09b..7aeb808 100644 --- a/api/configuration/configuration.go +++ b/api/configuration/configuration.go @@ -59,6 +59,8 @@ type Configuration struct { CacheTTL string `json:"cache_ttl"` ValidSubscription bool `json:"valid_subscription"` + + ReportDbUri string `json:"report_db_uri"` } var Config = Configuration{} @@ -242,4 +244,11 @@ func Init() { } else { Config.ValidSubscription = false } + + if os.Getenv("REPORT_DB_URI") != "" { + Config.ReportDbUri = os.Getenv("REPORT_DB_URI") + } else { + logs.Logs.Println("[CRITICAL][ENV] REPORT_DB_URI variable is empty") + os.Exit(1) + } } diff --git a/api/go.mod b/api/go.mod index 2902e3f..12616d6 100644 --- a/api/go.mod +++ b/api/go.mod @@ -10,7 +10,10 @@ require ( github.com/gin-contrib/cors v1.4.0 github.com/gin-contrib/gzip v0.0.6 github.com/gin-gonic/gin v1.9.1 + github.com/jackc/pgx/v5 v5.6.0 + github.com/mattn/go-sqlite3 v1.14.22 github.com/nqd/flat v0.2.0 + golang.org/x/crypto v0.17.0 ) require ( @@ -25,23 +28,24 @@ require ( github.com/goccy/go-json v0.10.2 // indirect github.com/golang-jwt/jwt/v4 v4.4.3 // indirect github.com/imdario/mergo v0.3.12 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect + github.com/jackc/puddle/v2 v2.2.1 // indirect github.com/jellydator/ttlcache/v3 v3.2.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/cpuid/v2 v2.2.4 // indirect github.com/leodido/go-urn v1.2.4 // indirect github.com/mattn/go-isatty v0.0.19 // indirect - github.com/mattn/go-sqlite3 v1.14.22 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pelletier/go-toml/v2 v2.0.8 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.11 // indirect golang.org/x/arch v0.3.0 // indirect - golang.org/x/crypto v0.9.0 // indirect golang.org/x/net v0.10.0 // indirect golang.org/x/sync v0.1.0 // indirect - golang.org/x/sys v0.8.0 // indirect - golang.org/x/text v0.9.0 // indirect + golang.org/x/sys v0.15.0 // indirect + golang.org/x/text v0.14.0 // indirect google.golang.org/protobuf v1.30.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/api/go.sum b/api/go.sum index 0555cb9..eca81b8 100644 --- a/api/go.sum +++ b/api/go.sum @@ -131,6 +131,14 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.6.0 h1:SWJzexBzPL5jb0GEsrPMLIsi/3jOo7RHlzTjcAeDrPY= +github.com/jackc/pgx/v5 v5.6.0/go.mod h1:DNZ/vlrUnhWCoFGxHAG8U2ljioxukquj7utPDgtQdTw= +github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= +github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jellydator/ttlcache/v3 v3.2.0 h1:6lqVJ8X3ZaUwvzENqPAobDsXNExfUJd61u++uW8a3LE= github.com/jellydator/ttlcache/v3 v3.2.0/go.mod h1:hi7MGFdMAwZna5n2tuvh63DvFLzVKySzCVW6+0gA2n4= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= @@ -276,6 +284,8 @@ golang.org/x/crypto v0.4.0/go.mod h1:3quD/ATkf6oY+rnes5c3ExXTbLc8mueNue5/DoinL80 golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU= golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g= golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= +golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= +golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -342,6 +352,8 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= @@ -359,6 +371,8 @@ golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190606050223-4d9ae51c2468/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190611222205-d73e1c7e250b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= diff --git a/api/main.go b/api/main.go index 08ee3e1..0c2b06e 100644 --- a/api/main.go +++ b/api/main.go @@ -131,6 +131,13 @@ func main() { } } + // report APIs + reports := router.Group("/reports") + reports.Use(middleware.ReportAuth()) + { + reports.POST("/mwan-series", methods.UpdateMwanSeries) + } + // handle missing endpoint router.NoRoute(func(c *gin.Context) { c.JSON(http.StatusNotFound, structs.Map(response.StatusNotFound{ diff --git a/api/middleware/middleware.go b/api/middleware/middleware.go index ec5dde2..f9aa107 100644 --- a/api/middleware/middleware.go +++ b/api/middleware/middleware.go @@ -14,6 +14,7 @@ import ( "encoding/json" "io" "io/ioutil" + "net/http" "strings" "time" @@ -257,3 +258,33 @@ func InitJWT() *jwt.GinJWTMiddleware { // return object return authMiddleware } + +func ReportAuth() gin.HandlerFunc { + return func(c *gin.Context) { + + token := c.GetHeader("RegistrationToken") + unit_id := c.GetHeader("UnitID") + if token == "" || unit_id == "" { + c.JSON(http.StatusUnauthorized, structs.Map(response.StatusUnauthorized{ + Code: 401, + Message: "missing token or unit id", + Data: nil, + })) + c.Abort() + return + } + valid := true + // TODO: check if token is valid + if !valid { + c.JSON(http.StatusUnauthorized, structs.Map(response.StatusUnauthorized{ + Code: 401, + Message: "invalid token or unit id", + Data: nil, + })) + c.Abort() + return + } + c.Next() + } + +} diff --git a/api/storage/report_schema.sql b/api/storage/report_schema.sql new file mode 100644 index 0000000..babb081 --- /dev/null +++ b/api/storage/report_schema.sql @@ -0,0 +1,14 @@ +/* + * Copyright (C) 2024 Nethesis S.r.l. + * http://www.nethesis.it - info@nethesis.it + * + * SPDX-License-Identifier: GPL-2.0-only + * + * author: Giacomo Sanchietti + */ + + CREATE TABLE IF NOT EXISTS units ( + id SERIAL PRIMARY KEY, + uuid TEXT, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP +); \ No newline at end of file diff --git a/api/storage/storage.go b/api/storage/storage.go index de1b5cc..a2141e1 100644 --- a/api/storage/storage.go +++ b/api/storage/storage.go @@ -10,6 +10,7 @@ package storage import ( + "context" "database/sql" _ "embed" "os" @@ -19,16 +20,21 @@ import ( "github.com/NethServer/nethsecurity-controller/api/logs" "github.com/NethServer/nethsecurity-controller/api/models" "github.com/NethServer/nethsecurity-controller/api/utils" + "github.com/jackc/pgx/v5/pgxpool" _ "github.com/mattn/go-sqlite3" ) var db *sql.DB +var dbpool *pgxpool.Pool var err error //go:embed schema.sql var schemaSQL string +//go:embed report_schema.sql +var reportSchemaSQL string + func Instance() *sql.DB { if db == nil { db = Init() @@ -262,3 +268,46 @@ func UpdatePassword(accountUsername string, newPassword string) error { return err } + +func InitReportDb() *pgxpool.Pool { + ctx := context.Background() + dbpool, err = pgxpool.New(ctx, configuration.Config.ReportDbUri) + if err != nil { + logs.Logs.Println("[CRITICAL][DB] error in db connection:" + err.Error()) + os.Exit(1) + } + + err = dbpool.Ping(ctx) + if err != nil { + logs.Logs.Println("[CRITICAL][DB] error in db connection:" + err.Error()) + os.Exit(1) + } + + // execute create tables + _, errExecute := db.Exec(reportSchemaSQL) + if errExecute != nil { + logs.Logs.Println("[ERR][STORAGE] error in storage file schema init:" + errExecute.Error()) + } + + // check if units table exists, create if not + _, err = dbpool.Exec(ctx, "CREATE TABLE IF NOT EXISTS units (id SERIAL PRIMARY KEY, uuid TEXT, created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP)") + + // mwan time series table + // { + // "timestamp": 1724221238, + // "wan": "wan", + // "interface": "eth1", + // "event": "offline" + // }, + + _, err = dbpool.Exec(ctx, "CREATE TABLE IF NOT EXISTS mwan_series (id SERIAL PRIMARY KEY, timestamp BIGINT, wan TEXT, interface TEXT, event TEXT)") + + return dbpool +} + +func ReportInstance() *pgxpool.Pool { + if dbpool == nil { + dbpool = InitReportDb() + } + return dbpool +}