From 932e4e119e7b6b33143839be818dbea1542cf106 Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Tue, 2 Apr 2024 11:53:14 +0200 Subject: [PATCH 1/8] api: list units, add cache parameter --- api/README.md | 4 ++++ api/cache/cache.go | 3 --- api/methods/unit.go | 20 +++++++++++++------- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/api/README.md b/api/README.md index d511a7a..069885c 100644 --- a/api/README.md +++ b/api/README.md @@ -158,6 +158,10 @@ CGO_ENABLED=0 go build "message": "units listed successfully" } ``` + + The API takes a query parameter `cache`. If `cache` is set to `true`, the API will return the cached data, if data are fresh enough. + If `cache` is set to `false`, the API will always fetch the data from the connected units. + - `GET /units/` REQ diff --git a/api/cache/cache.go b/api/cache/cache.go index 7d5c642..4322c70 100644 --- a/api/cache/cache.go +++ b/api/cache/cache.go @@ -35,20 +35,17 @@ func Init() { } func SetUnitInfo(unitId string, unitInfo models.UnitInfo) { - print("SET_CACHE" + unitId + "\n") value, err := strconv.Atoi(configuration.Config.CacheTTL) if err != nil { value = 60 } b, err := json.Marshal(unitInfo) - print("SET_CACHE" + string(b) + "\n") if err == nil { Cache.Set(unitId, string(b), time.Duration(value)*time.Second) } } func GetUnitInfo(unitId string) (models.UnitInfo, error) { - print("searching for unit info in cache " + unitId + "\n") if Cache.Has(unitId) { data := models.UnitInfo{} item := Cache.Get(unitId) diff --git a/api/methods/unit.go b/api/methods/unit.go index 4120d73..24a01d1 100644 --- a/api/methods/unit.go +++ b/api/methods/unit.go @@ -32,6 +32,9 @@ import ( ) func GetUnits(c *gin.Context) { + // get cache query param + cache := c.DefaultQuery("cache", "true") + // execute status command on openvpn socket var lines []string outSocket := socket.Write("status 3") @@ -119,10 +122,9 @@ func GetUnits(c *gin.Context) { } else { result["info"] = gin.H{} } - print("QUI\n") // FIXME: drop info from db, delete table? // add info from unit - remote_info, err := GetUnitInfo(e.Name()) + remote_info, err := GetUnitInfo(e.Name(), cache == "true") if err == nil { result["info"] = remote_info } @@ -652,11 +654,13 @@ func GetUnitToken(unitId string) (string, string, error) { return loginResponse.Token, loginResponse.Expire, nil } -func GetUnitInfo(unitId string) (models.UnitInfo, error) { +func GetUnitInfo(unitId string, useCache bool) (models.UnitInfo, error) { - info, error := cache.GetUnitInfo(unitId) - if error == nil { - return info, nil + if useCache { + info, error := cache.GetUnitInfo(unitId) + if error == nil { + return info, nil + } } // get the unit token and execute the request @@ -710,7 +714,9 @@ func GetUnitInfo(unitId string) (models.UnitInfo, error) { // FIXME: read all units on register // FIXME: load unit info every hour - cache.SetUnitInfo(unitId, unitInfo.Data) + if useCache { + cache.SetUnitInfo(unitId, unitInfo.Data) + } return unitInfo.Data, nil } From 529a8d3b7ada3c114abbfa0f088a64e9f5039201 Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Tue, 2 Apr 2024 12:30:02 +0200 Subject: [PATCH 2/8] api: always read info from remote Changes: - use the same functions to read unit info - drop local units table --- api/README.md | 26 ++++--- api/methods/unit.go | 167 +++++++++++++---------------------------- api/storage/schema.sql | 9 --- api/storage/storage.go | 74 ------------------ 4 files changed, 66 insertions(+), 210 deletions(-) diff --git a/api/README.md b/api/README.md index 069885c..d4b77f9 100644 --- a/api/README.md +++ b/api/README.md @@ -121,7 +121,6 @@ CGO_ENABLED=0 go build "ipaddress": "172.23.21.3", "id": "", "netmask": "255.255.255.0", - "registered": true, "vpn": { "bytes_rcvd": "21830", "bytes_sent": "5641", @@ -130,12 +129,12 @@ CGO_ENABLED=0 go build "virtual_address": "172.23.21.3" }, "info": { - "unit_id": "fba703c1-6c2d-4d3d-9dab-5998c7b66700", - "unit_name": "fw.local", + "unit_name": "myfw1", "version": "8-23.05.2-ns.0.0.2-beta2-37-g6e74afc", "subscription_type": "enterprise", "system_id": "XXXXXXXX-XXXX", - "created": "2024-03-14T15:18:08Z" + "ssh_port": 22, + "fqdn": "fw.local", } }, ... @@ -143,15 +142,14 @@ CGO_ENABLED=0 go build "ipaddress": "", "id": "", "netmask": "", - "registered": false, "vpn": {}, "info": { - "unit_id": "zzzzzzzz-d9f3-44b7-b277-36d65cf139e6", - "unit_name": "fw.nethsecurity.local", - "version": "8-23.05.2-ns.0.0.2-beta2-37-g6e74afc", + "unit_name": "", + "version": "", "subscription_type": "", "system_id": "", - "created": "2024-03-14T15:16:02Z" + "ssh_port": 0, + "fqdn": "", } } ], @@ -190,18 +188,22 @@ CGO_ENABLED=0 go build "virtual_address": "172.23.21.3" }, "info": { - "unit_id": "fba703c1-6c2d-4d3d-9dab-5998c7b66700", - "unit_name": "fw.local", + "unit_name": "myfw1", "version": "8-23.05.2-ns.0.0.2-beta2-37-g6e74afc", "subscription_type": "enterprise", "system_id": "XXXXXXXX-XXXX", - "created": "2024-03-14T15:18:08Z" + "ssh_port": 22, + "fqdn": "fw.local", }, "join_code": "eyJmcWRuIjoiY29udHJvbGxlci5ncy5uZXRoc2VydmVyLm5ldCIsInRva2VuIjoiMTIzNCIsInVuaXRfaWQiOiI5Njk0Y2Y4ZC03ZmE5LTRmN2EtYjFjNC1iY2Y0MGUzMjhjMDIifQ==" }, "message": "unit listed successfully" } ``` + + The API takes a query parameter `cache`. If `cache` is set to `true`, the API will return the cached data, if data are fresh enough. + If `cache` is set to `false`, the API will always fetch the data from the connected units. + - `GET /units//token` REQ diff --git a/api/methods/unit.go b/api/methods/unit.go index 24a01d1..66fd052 100644 --- a/api/methods/unit.go +++ b/api/methods/unit.go @@ -24,16 +24,13 @@ import ( "github.com/NethServer/nethsecurity-controller/api/configuration" "github.com/NethServer/nethsecurity-controller/api/models" "github.com/NethServer/nethsecurity-controller/api/socket" - "github.com/NethServer/nethsecurity-controller/api/storage" "github.com/NethServer/nethsecurity-controller/api/utils" "github.com/fatih/structs" "github.com/gin-gonic/gin" ) -func GetUnits(c *gin.Context) { - // get cache query param - cache := c.DefaultQuery("cache", "true") +func getVpnInfo() map[string]gin.H { // execute status command on openvpn socket var lines []string @@ -65,6 +62,15 @@ func GetUnits(c *gin.Context) { "connected_since": parts[8], } } + return vpns +} + +func GetUnits(c *gin.Context) { + // get cache query param + cache := c.DefaultQuery("cache", "true") + + // get vpn info + vpns := getVpnInfo() // list file in OpenVPNCCDDir units, err := os.ReadDir(configuration.Config.OpenVPNCCDDir) @@ -77,60 +83,19 @@ func GetUnits(c *gin.Context) { return } - // get unit data from database - unitRows, _ := storage.GetUnits() - dbInfo := make(map[string]models.Unit) - for _, unitRow := range unitRows { - dbInfo[unitRow.ID] = unitRow - } - // loop through units var results []gin.H for _, e := range units { // read unit file - unitFile, err := os.ReadFile(configuration.Config.OpenVPNCCDDir + "/" + e.Name()) + result, err := getUnitInfo(e.Name(), vpns, cache == "true") if err != nil { c.JSON(http.StatusBadRequest, structs.Map(response.StatusBadRequest{ Code: 400, - Message: "access CCD directory unit file failed", + Message: "Can't get unit info for: " + e.Name(), Data: err.Error(), })) } - // parse unit file - parts := strings.Split(string(unitFile), "\n") - parts = strings.Split(parts[0], " ") - - // compose result - result := gin.H{ - "id": e.Name(), - "ipaddress": parts[1], - "netmask": parts[2], - } - - // check if vpn data exists - if vpns[e.Name()] != nil { - result["vpn"] = vpns[e.Name()] - } else { - result["vpn"] = gin.H{} - } - - // add db info - info, ok := dbInfo[e.Name()] - if ok { - result["info"] = info - } else { - result["info"] = gin.H{} - } - // FIXME: drop info from db, delete table? - // add info from unit - remote_info, err := GetUnitInfo(e.Name(), cache == "true") - if err == nil { - result["info"] = remote_info - } - - result["join_code"] = utils.GetJoinCode(e.Name()) - // append to array results = append(results, result) } @@ -141,49 +106,44 @@ func GetUnits(c *gin.Context) { Message: "units listed successfully", Data: results, })) - } -func readUnitFile(unitId string) ([]byte, gin.H, error) { - // execute status command on openvpn socket - var lines []string - outSocket := socket.Write("status 3") - - // get only necessary lines - rawLines := strings.Split(outSocket, "\n") - for _, line := range rawLines { - if strings.HasPrefix(line, "CLIENT_LIST\t"+unitId) { - lines = append(lines, line) - } +func getUnitInfo(unitId string, vpns map[string]gin.H, useCache bool) (gin.H, error) { + unitFile, err := readUnitFile(unitId) + if err != nil { + return gin.H{}, err } - // define vpns object - var vpn gin.H + result := parseUnitFile(unitId, unitFile) - // loop through lines - for _, line := range lines { + // add info from unit + remote_info, err := GetUnitInfo(unitId, useCache) + if err == nil { + result["info"] = remote_info + } - // get values from line - parts := strings.Split(line, "\t") + // add join code + result["join_code"] = utils.GetJoinCode(unitId) - // compose result - vpn = gin.H{ - "real_address": parts[2], - "virtual_address": parts[3], - "bytes_rcvd": parts[5], - "bytes_sent": parts[6], - "connected_since": parts[8], - } + // add vpn info + if vpns[unitId] != nil { + result["vpn"] = vpns[unitId] + } else { + result["vpn"] = gin.H{} } + return result, nil +} + +func readUnitFile(unitId string) ([]byte, error) { // read unit file unitFile, err := os.ReadFile(configuration.Config.OpenVPNCCDDir + "/" + unitId) // return results - return unitFile, vpn, err + return unitFile, err } -func parseUnitFile(unitId string, unitFile []byte, vpn gin.H) gin.H { +func parseUnitFile(unitId string, unitFile []byte) gin.H { // parse unit file parts := strings.Split(string(unitFile), "\n") parts = strings.Split(parts[0], " ") @@ -195,48 +155,36 @@ func parseUnitFile(unitId string, unitFile []byte, vpn gin.H) gin.H { "netmask": parts[2], } - // check if vpn data exists - if vpn != nil { - result["vpn"] = vpn - } else { - result["vpn"] = gin.H{} - } - - // retrieve unit info from database - info, err := storage.GetUnit(unitId) - if err == nil { - result["info"] = info - } else { - result["info"] = gin.H{} - } - - result["join_code"] = utils.GetJoinCode(unitId) return result } func GetUnit(c *gin.Context) { + // get cache query param + cache := c.DefaultQuery("cache", "true") + + // get vpn info + vpns := getVpnInfo() + // get unit id unitId := c.Param("unit_id") - // read unit file - unitFile, vpn, err := readUnitFile(unitId) + // parse unit file + result, err := getUnitInfo(unitId, vpns, cache == "true") + if err != nil { c.JSON(http.StatusBadRequest, structs.Map(response.StatusBadRequest{ Code: 400, - Message: "access CCD directory unit file failed", + Message: "Can't get unit info for: " + unitId, Data: err.Error(), })) + } else { + // return 200 OK with data + c.JSON(http.StatusOK, structs.Map(response.StatusOK{ + Code: 200, + Message: "unit listed successfully", + Data: result, + })) } - - // parse unit file - result := parseUnitFile(unitId, unitFile, vpn) - - // return 200 OK with data - c.JSON(http.StatusOK, structs.Map(response.StatusOK{ - Code: 200, - Message: "unit listed successfully", - Data: result, - })) } func GetToken(c *gin.Context) { @@ -508,16 +456,6 @@ func RegisterUnit(c *gin.Context) { return } - errAdd := storage.AddOrUpdateUnit(jsonRequest.UnitId, jsonRequest.UnitName, jsonRequest.Version, jsonRequest.SubscriptionType, jsonRequest.SystemId) - if errAdd != nil { - c.JSON(http.StatusBadRequest, structs.Map(response.StatusBadRequest{ - Code: 400, - Message: "cannot add update to database for: " + jsonRequest.UnitId, - Data: errAdd.Error(), - })) - return - } - // return 200 OK with data c.JSON(http.StatusOK, structs.Map(response.StatusOK{ Code: 200, @@ -710,7 +648,6 @@ func GetUnitInfo(unitId string, useCache bool) (models.UnitInfo, error) { } // save to cache - // FIXME: add option to GET /units to ignore the cache // FIXME: read all units on register // FIXME: load unit info every hour diff --git a/api/storage/schema.sql b/api/storage/schema.sql index d5498fe..64a07d5 100644 --- a/api/storage/schema.sql +++ b/api/storage/schema.sql @@ -13,13 +13,4 @@ CREATE TABLE accounts ( `password` TEXT NOT NULL, `display_name` TEXT, `created` TIMESTAMP NOT NULL -); - -CREATE TABLE units ( - `id` TEXT NOT NULL PRIMARY KEY, - `name` TEXT NOT NULL UNIQUE, - `version` TEXT NOT NULL, - `system_id` TEXT, - `subscription_type` TEXT, - `created` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ); \ No newline at end of file diff --git a/api/storage/storage.go b/api/storage/storage.go index 7fb7f84..de1b5cc 100644 --- a/api/storage/storage.go +++ b/api/storage/storage.go @@ -262,77 +262,3 @@ func UpdatePassword(accountUsername string, newPassword string) error { return err } - -func AddOrUpdateUnit(unitID string, unitName string, version string, subscriptionType string, systemID string) error { - // get db - db := Instance() - - // define query - _, err := db.Exec( - "REPLACE INTO units (id, name, version, subscription_type, system_id) VALUES (?, ?, ?, ?, ?)", - unitID, - unitName, - version, - subscriptionType, - systemID, - ) - - // check error - if err != nil { - logs.Logs.Println("[ERR][STORAGE][ADD_OR_UPDATE_UNIT] error in insert units query: " + err.Error()) - } - - return err -} - -func GetUnit(unitId string) (models.Unit, error) { - // get db - db := Instance() - - // define query - query := "SELECT id, name, version, subscription_type, system_id, created FROM units where id = ?" - rows, err := db.Query(query, unitId) - if err != nil { - logs.Logs.Println("[ERR][STORAGE][GET_UNIT] error in query execution:" + err.Error()) - } - defer rows.Close() - - // loop rows - var result models.Unit - for rows.Next() { - if err := rows.Scan(&result.ID, &result.Name, &result.Version, &result.SubscriptionType, &result.SystemID, &result.Created); err != nil { - logs.Logs.Println("[ERR][STORAGE][GET_UNIT] error in query row extraction" + err.Error()) - } - } - - // return results - return result, err -} - -func GetUnits() ([]models.Unit, error) { - // get db - db := Instance() - - // define query - query := "SELECT id, name, version, subscription_type, system_id, created FROM units" - rows, err := db.Query(query) - if err != nil { - logs.Logs.Println("[ERR][STORAGE][GET_UNITS] error in query execution:" + err.Error()) - } - defer rows.Close() - - // loop rows - var results []models.Unit - for rows.Next() { - var unitRow models.Unit - if err := rows.Scan(&unitRow.ID, &unitRow.Name, &unitRow.Version, &unitRow.SubscriptionType, &unitRow.SystemID, &unitRow.Created); err != nil { - logs.Logs.Println("[ERR][STORAGE][GET_UNITS] error in query row extraction" + err.Error()) - } - - // append results - results = append(results, unitRow) - } - - // return results - return results, err -} From bb2bf127a9662e4a7d8b1b75237976bac9676a6b Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Tue, 2 Apr 2024 15:33:37 +0200 Subject: [PATCH 3/8] api: refresh cache on register and every hour --- api/README.md | 3 + api/cache/cache.go | 24 +++++ api/configuration/configuration.go | 2 +- api/methods/unit.go | 121 ++++--------------------- api/models/ubus.go | 9 -- api/models/unit.go | 137 ++++++++++++++++++++++++++++- 6 files changed, 180 insertions(+), 116 deletions(-) diff --git a/api/README.md b/api/README.md index d4b77f9..80e1ed3 100644 --- a/api/README.md +++ b/api/README.md @@ -39,6 +39,9 @@ CGO_ENABLED=0 go build - `FQDN`: fully qualified domain name of the machine - *default*: `hostname -f` +- `CACHE_TTL`: cache time to live for unit information in seconds - *default*: `7200` (2 hours) + Unit information are fetched from the connected units. The cache is refreshed every hour. + ## APIs ### Auth - `POST /login` diff --git a/api/cache/cache.go b/api/cache/cache.go index 4322c70..fc81732 100644 --- a/api/cache/cache.go +++ b/api/cache/cache.go @@ -22,6 +22,28 @@ import ( var Cache *ttlcache.Cache[string, string] +func refreshCache() { + // load all units info into cache + units, err := models.ListUnits() + if err != nil { + return + } + + for _, unit := range units { + unitInfo, err := models.GetRemoteInfo(unit) + if err == nil { + SetUnitInfo(unit, unitInfo) + } + } +} + +func refreshCacheLoop() { + ticker := time.NewTicker(60 * time.Minute) + for range ticker.C { + refreshCache() + } +} + func Init() { value, err := strconv.Atoi(configuration.Config.CacheTTL) if err != nil { @@ -32,6 +54,8 @@ func Init() { ttlcache.WithTTL[string, string](time.Duration(value) * time.Second), ) go Cache.Start() // starts automatic expired item deletion + + go refreshCacheLoop() // starts cache refresh loop } func SetUnitInfo(unitId string, unitInfo models.UnitInfo) { diff --git a/api/configuration/configuration.go b/api/configuration/configuration.go index c6d40ff..52ac99e 100644 --- a/api/configuration/configuration.go +++ b/api/configuration/configuration.go @@ -226,6 +226,6 @@ func Init() { if os.Getenv("CACHE_TTL") != "" { Config.CacheTTL = os.Getenv("CACHE_TTL") } else { - Config.CacheTTL = "3600" + Config.CacheTTL = "7200" } } diff --git a/api/methods/unit.go b/api/methods/unit.go index 66fd052..82912be 100644 --- a/api/methods/unit.go +++ b/api/methods/unit.go @@ -10,9 +10,7 @@ package methods import ( - "bytes" "encoding/json" - "errors" "io/ioutil" "net/http" "os" @@ -73,11 +71,11 @@ func GetUnits(c *gin.Context) { vpns := getVpnInfo() // list file in OpenVPNCCDDir - units, err := os.ReadDir(configuration.Config.OpenVPNCCDDir) + units, err := models.ListUnits() if err != nil { c.JSON(http.StatusBadRequest, structs.Map(response.StatusBadRequest{ Code: 400, - Message: "access CCD directory failed", + Message: "can't list units", Data: err.Error(), })) return @@ -87,11 +85,11 @@ func GetUnits(c *gin.Context) { var results []gin.H for _, e := range units { // read unit file - result, err := getUnitInfo(e.Name(), vpns, cache == "true") + result, err := getUnitInfo(e, vpns, cache == "true") if err != nil { c.JSON(http.StatusBadRequest, structs.Map(response.StatusBadRequest{ Code: 400, - Message: "Can't get unit info for: " + e.Name(), + Message: "Can't get unit info for: " + e, Data: err.Error(), })) } @@ -191,7 +189,7 @@ func GetToken(c *gin.Context) { // get unit id unitId := c.Param("unit_id") - token, expire, err := GetUnitToken(unitId) + token, expire, err := models.GetUnitToken(unitId) if err != nil { c.JSON(http.StatusBadRequest, structs.Map(response.StatusBadRequest{ @@ -456,6 +454,12 @@ func RegisterUnit(c *gin.Context) { return } + // cache unit info + unitInfo, err := models.GetRemoteInfo(jsonRequest.UnitId) + if err != nil { + cache.SetUnitInfo(jsonRequest.UnitId, unitInfo) + } + // return 200 OK with data c.JSON(http.StatusOK, structs.Map(response.StatusOK{ Code: 200, @@ -543,55 +547,6 @@ func DeleteUnit(c *gin.Context) { })) } -func GetUnitToken(unitId string) (string, string, error) { - - // read credentials - var credentials models.LoginRequest - body, err := ioutil.ReadFile(configuration.Config.CredentialsDir + "/" + unitId) - if err != nil { - return "", "", errors.New("cannot open credentials file for: " + unitId) - } - - // convert json string to struct - json.Unmarshal(body, &credentials) - - // compose request URL - postURL := configuration.Config.ProxyProtocol + configuration.Config.ProxyHost + ":" + configuration.Config.ProxyPort + "/" + unitId + configuration.Config.LoginEndpoint - - // create request action - r, err := http.NewRequest("POST", postURL, bytes.NewBuffer(body)) - if err != nil { - return "", "", errors.New("cannot make request for: " + unitId) - } - - // set request header - r.Header.Add("Content-Type", "application/json") - - // make request - client := &http.Client{} - res, err := client.Do(r) - if err != nil { - return "", "", errors.New("request failed for: " + unitId) - } - - // close response - defer res.Body.Close() - - // convert response to struct - loginResponse := &models.LoginResponse{} - err = json.NewDecoder(res.Body).Decode(loginResponse) - if err != nil { - return "", "", errors.New("cannot convert response to struct for: " + unitId) - } - - // check if token is not empty - if len(loginResponse.Token) == 0 { - return "", "", errors.New("invalid JWT token response for: " + unitId) - } - - return loginResponse.Token, loginResponse.Expire, nil -} - func GetUnitInfo(unitId string, useCache bool) (models.UnitInfo, error) { if useCache { @@ -601,59 +556,15 @@ func GetUnitInfo(unitId string, useCache bool) (models.UnitInfo, error) { } } - // get the unit token and execute the request - token, _, _ := GetUnitToken(unitId) - if token == "" { - return models.UnitInfo{}, errors.New("error getting token") - } - - // compose request URL - postURL := configuration.Config.ProxyProtocol + configuration.Config.ProxyHost + ":" + configuration.Config.ProxyPort + "/" + unitId + "/api/ubus/call" - // prepare the payload: {"path":"ns.don","method":"status","payload":{}} - payload := models.UbusCommand{ - Path: "ns.controller", - Method: "info", - Payload: map[string]interface{}{}, - } - - // convert payload to JSON byte array - payloadBytes, err := json.Marshal(payload) + // get remote info + unitInfo, err := models.GetRemoteInfo(unitId) if err != nil { - return models.UnitInfo{}, errors.New("error marshalling payload") + return models.UnitInfo{}, err } - // create request action - r, err := http.NewRequest("POST", postURL, bytes.NewBuffer(payloadBytes)) - if err != nil { - return models.UnitInfo{}, errors.New("error creating request") - } - - // set request headers - r.Header.Add("Content-Type", "application/json") - r.Header.Add("Authorization", "Bearer "+token) - - // make request - client := &http.Client{} - res, err := client.Do(r) - if err != nil { - return models.UnitInfo{}, errors.New("error making request") - } - defer res.Body.Close() - - // convert response to struct - unitInfo := &models.UbusInfoResponse{} - err = json.NewDecoder(res.Body).Decode(unitInfo) - if err != nil { - return models.UnitInfo{}, errors.New("error decoding response") - } - - // save to cache - // FIXME: read all units on register - // FIXME: load unit info every hour - if useCache { - cache.SetUnitInfo(unitId, unitInfo.Data) + cache.SetUnitInfo(unitId, unitInfo) } - return unitInfo.Data, nil + return unitInfo, nil } diff --git a/api/models/ubus.go b/api/models/ubus.go index 9460ce0..319d0c7 100644 --- a/api/models/ubus.go +++ b/api/models/ubus.go @@ -22,12 +22,3 @@ type UbusInfoResponse struct { Data UnitInfo `json:"data"` Message string `json:"message"` } - -type UnitInfo struct { - UnitName string `json:"unit_name"` - Version string `json:"version"` - SubscriptionType string `json:"subscription_type"` - SystemID string `json:"system_id"` - SSHPort int `json:"ssh_port"` - FQDN string `json:"fqdn"` -} diff --git a/api/models/unit.go b/api/models/unit.go index 182ac9e..430d120 100644 --- a/api/models/unit.go +++ b/api/models/unit.go @@ -9,7 +9,17 @@ package models -import "time" +import ( + "bytes" + "encoding/json" + "errors" + "io/ioutil" + "net/http" + "os" + "time" + + "github.com/NethServer/nethsecurity-controller/api/configuration" +) type AddRequest struct { UnitId string `json:"unit_id" binding:"required"` @@ -33,3 +43,128 @@ type Unit struct { SystemID string `json:"system_id" structs:"system_id"` Created time.Time `json:"created" structs:"created"` } + +type UnitInfo struct { + UnitName string `json:"unit_name"` + Version string `json:"version"` + SubscriptionType string `json:"subscription_type"` + SystemID string `json:"system_id"` + SSHPort int `json:"ssh_port"` + FQDN string `json:"fqdn"` +} + +// list unit name from files in OpenVPNCCDDir +func ListUnits() ([]string, error) { + units := []string{} + // list file in OpenVPNCCDDir + files, err := os.ReadDir(configuration.Config.OpenVPNCCDDir) + if err != nil { + return nil, err + } + + // loop through files + for _, file := range files { + units = append(units, file.Name()) + } + + return units, nil +} + +func GetUnitToken(unitId string) (string, string, error) { + + // read credentials + var credentials LoginRequest + body, err := ioutil.ReadFile(configuration.Config.CredentialsDir + "/" + unitId) + if err != nil { + return "", "", errors.New("cannot open credentials file for: " + unitId) + } + + // convert json string to struct + json.Unmarshal(body, &credentials) + + // compose request URL + postURL := configuration.Config.ProxyProtocol + configuration.Config.ProxyHost + ":" + configuration.Config.ProxyPort + "/" + unitId + configuration.Config.LoginEndpoint + + // create request action + r, err := http.NewRequest("POST", postURL, bytes.NewBuffer(body)) + if err != nil { + return "", "", errors.New("cannot make request for: " + unitId) + } + + // set request header + r.Header.Add("Content-Type", "application/json") + + // make request + client := &http.Client{} + res, err := client.Do(r) + if err != nil { + return "", "", errors.New("request failed for: " + unitId) + } + + // close response + defer res.Body.Close() + + // convert response to struct + loginResponse := &LoginResponse{} + err = json.NewDecoder(res.Body).Decode(loginResponse) + if err != nil { + return "", "", errors.New("cannot convert response to struct for: " + unitId) + } + + // check if token is not empty + if len(loginResponse.Token) == 0 { + return "", "", errors.New("invalid JWT token response for: " + unitId) + } + + return loginResponse.Token, loginResponse.Expire, nil +} + +func GetRemoteInfo(unitId string) (UnitInfo, error) { + // get the unit token and execute the request + token, _, _ := GetUnitToken(unitId) + if token == "" { + return UnitInfo{}, errors.New("error getting token") + } + + // compose request URL + postURL := configuration.Config.ProxyProtocol + configuration.Config.ProxyHost + ":" + configuration.Config.ProxyPort + "/" + unitId + "/api/ubus/call" + // prepare the payload: {"path":"ns.don","method":"status","payload":{}} + payload := UbusCommand{ + Path: "ns.controller", + Method: "info", + Payload: map[string]interface{}{}, + } + + // convert payload to JSON byte array + payloadBytes, err := json.Marshal(payload) + if err != nil { + return UnitInfo{}, errors.New("error marshalling payload") + } + + // create request action + r, err := http.NewRequest("POST", postURL, bytes.NewBuffer(payloadBytes)) + if err != nil { + return UnitInfo{}, errors.New("error creating request") + } + + // set request headers + r.Header.Add("Content-Type", "application/json") + r.Header.Add("Authorization", "Bearer "+token) + + // make request + client := &http.Client{} + res, err := client.Do(r) + if err != nil { + return UnitInfo{}, errors.New("error making request") + } + defer res.Body.Close() + + // convert response to struct + unitInfo := &UbusInfoResponse{} + err = json.NewDecoder(res.Body).Decode(unitInfo) + if err != nil { + return UnitInfo{}, errors.New("error decoding response") + } + + return unitInfo.Data, nil +} From d48b602d8a83a0e39086f12e8731e02bb03d4995 Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Tue, 2 Apr 2024 15:49:16 +0200 Subject: [PATCH 4/8] api: GetUnitInfo, always refresh cache If useCache flag is false, do not retrieve info from cache but always store the result to refresh the existing cache. --- api/methods/unit.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/api/methods/unit.go b/api/methods/unit.go index 82912be..20ca62d 100644 --- a/api/methods/unit.go +++ b/api/methods/unit.go @@ -562,9 +562,8 @@ func GetUnitInfo(unitId string, useCache bool) (models.UnitInfo, error) { return models.UnitInfo{}, err } - if useCache { - cache.SetUnitInfo(unitId, unitInfo) - } + // cache is always updated + cache.SetUnitInfo(unitId, unitInfo) return unitInfo, nil } From 5f3730505f2d8d55b12bf49380d8b9a440ed6f94 Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Tue, 2 Apr 2024 16:14:47 +0200 Subject: [PATCH 5/8] api: always return info block --- api/methods/unit.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/api/methods/unit.go b/api/methods/unit.go index 20ca62d..87197fe 100644 --- a/api/methods/unit.go +++ b/api/methods/unit.go @@ -118,6 +118,8 @@ func getUnitInfo(unitId string, vpns map[string]gin.H, useCache bool) (gin.H, er remote_info, err := GetUnitInfo(unitId, useCache) if err == nil { result["info"] = remote_info + } else { + result["info"] = gin.H{} } // add join code From 0742571edb4925140629fc8731e1fd521d7213e2 Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Tue, 2 Apr 2024 16:15:04 +0200 Subject: [PATCH 6/8] api: add request timeout for remote unit info Now the timeout is 4 seconds: - 2 seconds for the login - 2 seconds for the controller API --- api/models/unit.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/api/models/unit.go b/api/models/unit.go index 430d120..3f53907 100644 --- a/api/models/unit.go +++ b/api/models/unit.go @@ -94,8 +94,8 @@ func GetUnitToken(unitId string) (string, string, error) { // set request header r.Header.Add("Content-Type", "application/json") - // make request - client := &http.Client{} + // make request, 2 seconds timeout + client := &http.Client{Timeout: 2 * time.Second} res, err := client.Do(r) if err != nil { return "", "", errors.New("request failed for: " + unitId) @@ -151,8 +151,8 @@ func GetRemoteInfo(unitId string) (UnitInfo, error) { r.Header.Add("Content-Type", "application/json") r.Header.Add("Authorization", "Bearer "+token) - // make request - client := &http.Client{} + // make request, with 2 seconds timeout + client := &http.Client{Timeout: 2 * time.Second} res, err := client.Do(r) if err != nil { return UnitInfo{}, errors.New("error making request") From 4f5285ecfd6661d0c660ddf94c0cdafc81e38c60 Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Tue, 2 Apr 2024 17:10:45 +0200 Subject: [PATCH 7/8] api: do not retrieve info on register After the registration, the unit may take a while to establish the VPN --- api/methods/unit.go | 8 +------- api/models/unit.go | 2 +- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/api/methods/unit.go b/api/methods/unit.go index 87197fe..9ea0407 100644 --- a/api/methods/unit.go +++ b/api/methods/unit.go @@ -439,7 +439,7 @@ func RegisterUnit(c *gin.Context) { credentials.Password = password } } else { - // update credentials + // create credentials credentials.Username = username credentials.Password = password } @@ -456,12 +456,6 @@ func RegisterUnit(c *gin.Context) { return } - // cache unit info - unitInfo, err := models.GetRemoteInfo(jsonRequest.UnitId) - if err != nil { - cache.SetUnitInfo(jsonRequest.UnitId, unitInfo) - } - // return 200 OK with data c.JSON(http.StatusOK, structs.Map(response.StatusOK{ Code: 200, diff --git a/api/models/unit.go b/api/models/unit.go index 3f53907..5816cdd 100644 --- a/api/models/unit.go +++ b/api/models/unit.go @@ -113,7 +113,7 @@ func GetUnitToken(unitId string) (string, string, error) { // check if token is not empty if len(loginResponse.Token) == 0 { - return "", "", errors.New("invalid JWT token response for: " + unitId) + return "", "", errors.New("invalid token response for: " + unitId) } return loginResponse.Token, loginResponse.Expire, nil From 61f9287d119ee6781b2a382baec5abf77071e59e Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Wed, 3 Apr 2024 10:11:27 +0200 Subject: [PATCH 8/8] api: refactor code Changes: - move units methods to methods package - move cache initialization to main to avoid import loop --- api/cache/cache.go | 26 +-------- api/main.go | 21 ++++++++ api/methods/unit.go | 125 ++++++++++++++++++++++++++++++++++++++++++-- api/models/unit.go | 124 ------------------------------------------- 4 files changed, 144 insertions(+), 152 deletions(-) diff --git a/api/cache/cache.go b/api/cache/cache.go index fc81732..97e2c89 100644 --- a/api/cache/cache.go +++ b/api/cache/cache.go @@ -22,40 +22,16 @@ import ( var Cache *ttlcache.Cache[string, string] -func refreshCache() { - // load all units info into cache - units, err := models.ListUnits() - if err != nil { - return - } - - for _, unit := range units { - unitInfo, err := models.GetRemoteInfo(unit) - if err == nil { - SetUnitInfo(unit, unitInfo) - } - } -} - -func refreshCacheLoop() { - ticker := time.NewTicker(60 * time.Minute) - for range ticker.C { - refreshCache() - } -} - func Init() { value, err := strconv.Atoi(configuration.Config.CacheTTL) if err != nil { value = 3600 } - Cache = ttlcache.New[string, string]( + Cache = ttlcache.New( ttlcache.WithTTL[string, string](time.Duration(value) * time.Second), ) go Cache.Start() // starts automatic expired item deletion - - go refreshCacheLoop() // starts cache refresh loop } func SetUnitInfo(unitId string, unitInfo models.UnitInfo) { diff --git a/api/main.go b/api/main.go index a5e4270..3b7e35d 100644 --- a/api/main.go +++ b/api/main.go @@ -12,6 +12,7 @@ package main import ( "io/ioutil" "net/http" + "time" "github.com/fatih/structs" "github.com/gin-contrib/cors" @@ -42,6 +43,24 @@ import ( // @schemes http // @BasePath /api +func refreshCacheLoop() { + ticker := time.NewTicker(60 * time.Minute) + for range ticker.C { + // load all units info into cache + units, err := methods.ListUnits() + if err != nil { + return + } + + for _, unit := range units { + unitInfo, err := methods.GetRemoteInfo(unit) + if err == nil { + cache.SetUnitInfo(unit, unitInfo) + } + } + } +} + func main() { // init logs with syslog logs.Init("nethsecurity_controller") @@ -58,6 +77,8 @@ func main() { // init cache cache.Init() + go refreshCacheLoop() // starts cache refresh loop + // disable log to stdout when running in release mode if gin.Mode() == gin.ReleaseMode { gin.DefaultWriter = ioutil.Discard diff --git a/api/methods/unit.go b/api/methods/unit.go index 9ea0407..71c03bc 100644 --- a/api/methods/unit.go +++ b/api/methods/unit.go @@ -10,12 +10,15 @@ package methods import ( + "bytes" "encoding/json" + "errors" "io/ioutil" "net/http" "os" "os/exec" "strings" + "time" "github.com/NethServer/nethsecurity-api/response" "github.com/NethServer/nethsecurity-controller/api/cache" @@ -71,7 +74,7 @@ func GetUnits(c *gin.Context) { vpns := getVpnInfo() // list file in OpenVPNCCDDir - units, err := models.ListUnits() + units, err := ListUnits() if err != nil { c.JSON(http.StatusBadRequest, structs.Map(response.StatusBadRequest{ Code: 400, @@ -191,7 +194,7 @@ func GetToken(c *gin.Context) { // get unit id unitId := c.Param("unit_id") - token, expire, err := models.GetUnitToken(unitId) + token, expire, err := GetUnitToken(unitId) if err != nil { c.JSON(http.StatusBadRequest, structs.Map(response.StatusBadRequest{ @@ -553,7 +556,7 @@ func GetUnitInfo(unitId string, useCache bool) (models.UnitInfo, error) { } // get remote info - unitInfo, err := models.GetRemoteInfo(unitId) + unitInfo, err := GetRemoteInfo(unitId) if err != nil { return models.UnitInfo{}, err } @@ -563,3 +566,119 @@ func GetUnitInfo(unitId string, useCache bool) (models.UnitInfo, error) { return unitInfo, nil } + +// list unit name from files in OpenVPNCCDDir +func ListUnits() ([]string, error) { + units := []string{} + // list file in OpenVPNCCDDir + files, err := os.ReadDir(configuration.Config.OpenVPNCCDDir) + if err != nil { + return nil, err + } + + // loop through files + for _, file := range files { + units = append(units, file.Name()) + } + + return units, nil +} + +func GetUnitToken(unitId string) (string, string, error) { + + // read credentials + var credentials models.LoginRequest + body, err := ioutil.ReadFile(configuration.Config.CredentialsDir + "/" + unitId) + if err != nil { + return "", "", errors.New("cannot open credentials file for: " + unitId) + } + + // convert json string to struct + json.Unmarshal(body, &credentials) + + // compose request URL + postURL := configuration.Config.ProxyProtocol + configuration.Config.ProxyHost + ":" + configuration.Config.ProxyPort + "/" + unitId + configuration.Config.LoginEndpoint + + // create request action + r, err := http.NewRequest("POST", postURL, bytes.NewBuffer(body)) + if err != nil { + return "", "", errors.New("cannot make request for: " + unitId) + } + + // set request header + r.Header.Add("Content-Type", "application/json") + + // make request, 2 seconds timeout + client := &http.Client{Timeout: 2 * time.Second} + res, err := client.Do(r) + if err != nil { + return "", "", errors.New("request failed for: " + unitId) + } + + // close response + defer res.Body.Close() + + // convert response to struct + loginResponse := &models.LoginResponse{} + err = json.NewDecoder(res.Body).Decode(loginResponse) + if err != nil { + return "", "", errors.New("cannot convert response to struct for: " + unitId) + } + + // check if token is not empty + if len(loginResponse.Token) == 0 { + return "", "", errors.New("invalid token response for: " + unitId) + } + + return loginResponse.Token, loginResponse.Expire, nil +} + +func GetRemoteInfo(unitId string) (models.UnitInfo, error) { + // get the unit token and execute the request + token, _, _ := GetUnitToken(unitId) + if token == "" { + return models.UnitInfo{}, errors.New("error getting token") + } + + // compose request URL + postURL := configuration.Config.ProxyProtocol + configuration.Config.ProxyHost + ":" + configuration.Config.ProxyPort + "/" + unitId + "/api/ubus/call" + // prepare the payload: {"path":"ns.don","method":"status","payload":{}} + payload := models.UbusCommand{ + Path: "ns.controller", + Method: "info", + Payload: map[string]interface{}{}, + } + + // convert payload to JSON byte array + payloadBytes, err := json.Marshal(payload) + if err != nil { + return models.UnitInfo{}, errors.New("error marshalling payload") + } + + // create request action + r, err := http.NewRequest("POST", postURL, bytes.NewBuffer(payloadBytes)) + if err != nil { + return models.UnitInfo{}, errors.New("error creating request") + } + + // set request headers + r.Header.Add("Content-Type", "application/json") + r.Header.Add("Authorization", "Bearer "+token) + + // make request, with 2 seconds timeout + client := &http.Client{Timeout: 2 * time.Second} + res, err := client.Do(r) + if err != nil { + return models.UnitInfo{}, errors.New("error making request") + } + defer res.Body.Close() + + // convert response to struct + unitInfo := &models.UbusInfoResponse{} + err = json.NewDecoder(res.Body).Decode(unitInfo) + if err != nil { + return models.UnitInfo{}, errors.New("error decoding response") + } + + return unitInfo.Data, nil +} diff --git a/api/models/unit.go b/api/models/unit.go index 5816cdd..f0e8765 100644 --- a/api/models/unit.go +++ b/api/models/unit.go @@ -10,15 +10,7 @@ package models import ( - "bytes" - "encoding/json" - "errors" - "io/ioutil" - "net/http" - "os" "time" - - "github.com/NethServer/nethsecurity-controller/api/configuration" ) type AddRequest struct { @@ -52,119 +44,3 @@ type UnitInfo struct { SSHPort int `json:"ssh_port"` FQDN string `json:"fqdn"` } - -// list unit name from files in OpenVPNCCDDir -func ListUnits() ([]string, error) { - units := []string{} - // list file in OpenVPNCCDDir - files, err := os.ReadDir(configuration.Config.OpenVPNCCDDir) - if err != nil { - return nil, err - } - - // loop through files - for _, file := range files { - units = append(units, file.Name()) - } - - return units, nil -} - -func GetUnitToken(unitId string) (string, string, error) { - - // read credentials - var credentials LoginRequest - body, err := ioutil.ReadFile(configuration.Config.CredentialsDir + "/" + unitId) - if err != nil { - return "", "", errors.New("cannot open credentials file for: " + unitId) - } - - // convert json string to struct - json.Unmarshal(body, &credentials) - - // compose request URL - postURL := configuration.Config.ProxyProtocol + configuration.Config.ProxyHost + ":" + configuration.Config.ProxyPort + "/" + unitId + configuration.Config.LoginEndpoint - - // create request action - r, err := http.NewRequest("POST", postURL, bytes.NewBuffer(body)) - if err != nil { - return "", "", errors.New("cannot make request for: " + unitId) - } - - // set request header - r.Header.Add("Content-Type", "application/json") - - // make request, 2 seconds timeout - client := &http.Client{Timeout: 2 * time.Second} - res, err := client.Do(r) - if err != nil { - return "", "", errors.New("request failed for: " + unitId) - } - - // close response - defer res.Body.Close() - - // convert response to struct - loginResponse := &LoginResponse{} - err = json.NewDecoder(res.Body).Decode(loginResponse) - if err != nil { - return "", "", errors.New("cannot convert response to struct for: " + unitId) - } - - // check if token is not empty - if len(loginResponse.Token) == 0 { - return "", "", errors.New("invalid token response for: " + unitId) - } - - return loginResponse.Token, loginResponse.Expire, nil -} - -func GetRemoteInfo(unitId string) (UnitInfo, error) { - // get the unit token and execute the request - token, _, _ := GetUnitToken(unitId) - if token == "" { - return UnitInfo{}, errors.New("error getting token") - } - - // compose request URL - postURL := configuration.Config.ProxyProtocol + configuration.Config.ProxyHost + ":" + configuration.Config.ProxyPort + "/" + unitId + "/api/ubus/call" - // prepare the payload: {"path":"ns.don","method":"status","payload":{}} - payload := UbusCommand{ - Path: "ns.controller", - Method: "info", - Payload: map[string]interface{}{}, - } - - // convert payload to JSON byte array - payloadBytes, err := json.Marshal(payload) - if err != nil { - return UnitInfo{}, errors.New("error marshalling payload") - } - - // create request action - r, err := http.NewRequest("POST", postURL, bytes.NewBuffer(payloadBytes)) - if err != nil { - return UnitInfo{}, errors.New("error creating request") - } - - // set request headers - r.Header.Add("Content-Type", "application/json") - r.Header.Add("Authorization", "Bearer "+token) - - // make request, with 2 seconds timeout - client := &http.Client{Timeout: 2 * time.Second} - res, err := client.Do(r) - if err != nil { - return UnitInfo{}, errors.New("error making request") - } - defer res.Body.Close() - - // convert response to struct - unitInfo := &UbusInfoResponse{} - err = json.NewDecoder(res.Body).Decode(unitInfo) - if err != nil { - return UnitInfo{}, errors.New("error decoding response") - } - - return unitInfo.Data, nil -}