diff --git a/api/README.md b/api/README.md index d511a7a..80e1ed3 100644 --- a/api/README.md +++ b/api/README.md @@ -39,6 +39,9 @@ CGO_ENABLED=0 go build - `FQDN`: fully qualified domain name of the machine - *default*: `hostname -f` +- `CACHE_TTL`: cache time to live for unit information in seconds - *default*: `7200` (2 hours) + Unit information are fetched from the connected units. The cache is refreshed every hour. + ## APIs ### Auth - `POST /login` @@ -121,7 +124,6 @@ CGO_ENABLED=0 go build "ipaddress": "172.23.21.3", "id": "", "netmask": "255.255.255.0", - "registered": true, "vpn": { "bytes_rcvd": "21830", "bytes_sent": "5641", @@ -130,12 +132,12 @@ CGO_ENABLED=0 go build "virtual_address": "172.23.21.3" }, "info": { - "unit_id": "fba703c1-6c2d-4d3d-9dab-5998c7b66700", - "unit_name": "fw.local", + "unit_name": "myfw1", "version": "8-23.05.2-ns.0.0.2-beta2-37-g6e74afc", "subscription_type": "enterprise", "system_id": "XXXXXXXX-XXXX", - "created": "2024-03-14T15:18:08Z" + "ssh_port": 22, + "fqdn": "fw.local", } }, ... @@ -143,21 +145,24 @@ CGO_ENABLED=0 go build "ipaddress": "", "id": "", "netmask": "", - "registered": false, "vpn": {}, "info": { - "unit_id": "zzzzzzzz-d9f3-44b7-b277-36d65cf139e6", - "unit_name": "fw.nethsecurity.local", - "version": "8-23.05.2-ns.0.0.2-beta2-37-g6e74afc", + "unit_name": "", + "version": "", "subscription_type": "", "system_id": "", - "created": "2024-03-14T15:16:02Z" + "ssh_port": 0, + "fqdn": "", } } ], "message": "units listed successfully" } ``` + + The API takes a query parameter `cache`. If `cache` is set to `true`, the API will return the cached data, if data are fresh enough. + If `cache` is set to `false`, the API will always fetch the data from the connected units. + - `GET /units/` REQ @@ -186,18 +191,22 @@ CGO_ENABLED=0 go build "virtual_address": "172.23.21.3" }, "info": { - "unit_id": "fba703c1-6c2d-4d3d-9dab-5998c7b66700", - "unit_name": "fw.local", + "unit_name": "myfw1", "version": "8-23.05.2-ns.0.0.2-beta2-37-g6e74afc", "subscription_type": "enterprise", "system_id": "XXXXXXXX-XXXX", - "created": "2024-03-14T15:18:08Z" + "ssh_port": 22, + "fqdn": "fw.local", }, "join_code": "eyJmcWRuIjoiY29udHJvbGxlci5ncy5uZXRoc2VydmVyLm5ldCIsInRva2VuIjoiMTIzNCIsInVuaXRfaWQiOiI5Njk0Y2Y4ZC03ZmE5LTRmN2EtYjFjNC1iY2Y0MGUzMjhjMDIifQ==" }, "message": "unit listed successfully" } ``` + + The API takes a query parameter `cache`. If `cache` is set to `true`, the API will return the cached data, if data are fresh enough. + If `cache` is set to `false`, the API will always fetch the data from the connected units. + - `GET /units//token` REQ diff --git a/api/cache/cache.go b/api/cache/cache.go index 7d5c642..97e2c89 100644 --- a/api/cache/cache.go +++ b/api/cache/cache.go @@ -28,27 +28,24 @@ func Init() { value = 3600 } - Cache = ttlcache.New[string, string]( + Cache = ttlcache.New( ttlcache.WithTTL[string, string](time.Duration(value) * time.Second), ) go Cache.Start() // starts automatic expired item deletion } func SetUnitInfo(unitId string, unitInfo models.UnitInfo) { - print("SET_CACHE" + unitId + "\n") value, err := strconv.Atoi(configuration.Config.CacheTTL) if err != nil { value = 60 } b, err := json.Marshal(unitInfo) - print("SET_CACHE" + string(b) + "\n") if err == nil { Cache.Set(unitId, string(b), time.Duration(value)*time.Second) } } func GetUnitInfo(unitId string) (models.UnitInfo, error) { - print("searching for unit info in cache " + unitId + "\n") if Cache.Has(unitId) { data := models.UnitInfo{} item := Cache.Get(unitId) diff --git a/api/configuration/configuration.go b/api/configuration/configuration.go index c6d40ff..52ac99e 100644 --- a/api/configuration/configuration.go +++ b/api/configuration/configuration.go @@ -226,6 +226,6 @@ func Init() { if os.Getenv("CACHE_TTL") != "" { Config.CacheTTL = os.Getenv("CACHE_TTL") } else { - Config.CacheTTL = "3600" + Config.CacheTTL = "7200" } } diff --git a/api/main.go b/api/main.go index a5e4270..3b7e35d 100644 --- a/api/main.go +++ b/api/main.go @@ -12,6 +12,7 @@ package main import ( "io/ioutil" "net/http" + "time" "github.com/fatih/structs" "github.com/gin-contrib/cors" @@ -42,6 +43,24 @@ import ( // @schemes http // @BasePath /api +func refreshCacheLoop() { + ticker := time.NewTicker(60 * time.Minute) + for range ticker.C { + // load all units info into cache + units, err := methods.ListUnits() + if err != nil { + return + } + + for _, unit := range units { + unitInfo, err := methods.GetRemoteInfo(unit) + if err == nil { + cache.SetUnitInfo(unit, unitInfo) + } + } + } +} + func main() { // init logs with syslog logs.Init("nethsecurity_controller") @@ -58,6 +77,8 @@ func main() { // init cache cache.Init() + go refreshCacheLoop() // starts cache refresh loop + // disable log to stdout when running in release mode if gin.Mode() == gin.ReleaseMode { gin.DefaultWriter = ioutil.Discard diff --git a/api/methods/unit.go b/api/methods/unit.go index 4120d73..71c03bc 100644 --- a/api/methods/unit.go +++ b/api/methods/unit.go @@ -18,20 +18,21 @@ import ( "os" "os/exec" "strings" + "time" "github.com/NethServer/nethsecurity-api/response" "github.com/NethServer/nethsecurity-controller/api/cache" "github.com/NethServer/nethsecurity-controller/api/configuration" "github.com/NethServer/nethsecurity-controller/api/models" "github.com/NethServer/nethsecurity-controller/api/socket" - "github.com/NethServer/nethsecurity-controller/api/storage" "github.com/NethServer/nethsecurity-controller/api/utils" "github.com/fatih/structs" "github.com/gin-gonic/gin" ) -func GetUnits(c *gin.Context) { +func getVpnInfo() map[string]gin.H { + // execute status command on openvpn socket var lines []string outSocket := socket.Write("status 3") @@ -62,73 +63,40 @@ func GetUnits(c *gin.Context) { "connected_since": parts[8], } } + return vpns +} + +func GetUnits(c *gin.Context) { + // get cache query param + cache := c.DefaultQuery("cache", "true") + + // get vpn info + vpns := getVpnInfo() // list file in OpenVPNCCDDir - units, err := os.ReadDir(configuration.Config.OpenVPNCCDDir) + units, err := ListUnits() if err != nil { c.JSON(http.StatusBadRequest, structs.Map(response.StatusBadRequest{ Code: 400, - Message: "access CCD directory failed", + Message: "can't list units", Data: err.Error(), })) return } - // get unit data from database - unitRows, _ := storage.GetUnits() - dbInfo := make(map[string]models.Unit) - for _, unitRow := range unitRows { - dbInfo[unitRow.ID] = unitRow - } - // loop through units var results []gin.H for _, e := range units { // read unit file - unitFile, err := os.ReadFile(configuration.Config.OpenVPNCCDDir + "/" + e.Name()) + result, err := getUnitInfo(e, vpns, cache == "true") if err != nil { c.JSON(http.StatusBadRequest, structs.Map(response.StatusBadRequest{ Code: 400, - Message: "access CCD directory unit file failed", + Message: "Can't get unit info for: " + e, Data: err.Error(), })) } - // parse unit file - parts := strings.Split(string(unitFile), "\n") - parts = strings.Split(parts[0], " ") - - // compose result - result := gin.H{ - "id": e.Name(), - "ipaddress": parts[1], - "netmask": parts[2], - } - - // check if vpn data exists - if vpns[e.Name()] != nil { - result["vpn"] = vpns[e.Name()] - } else { - result["vpn"] = gin.H{} - } - - // add db info - info, ok := dbInfo[e.Name()] - if ok { - result["info"] = info - } else { - result["info"] = gin.H{} - } - print("QUI\n") - // FIXME: drop info from db, delete table? - // add info from unit - remote_info, err := GetUnitInfo(e.Name()) - if err == nil { - result["info"] = remote_info - } - - result["join_code"] = utils.GetJoinCode(e.Name()) - // append to array results = append(results, result) } @@ -139,49 +107,46 @@ func GetUnits(c *gin.Context) { Message: "units listed successfully", Data: results, })) - } -func readUnitFile(unitId string) ([]byte, gin.H, error) { - // execute status command on openvpn socket - var lines []string - outSocket := socket.Write("status 3") - - // get only necessary lines - rawLines := strings.Split(outSocket, "\n") - for _, line := range rawLines { - if strings.HasPrefix(line, "CLIENT_LIST\t"+unitId) { - lines = append(lines, line) - } +func getUnitInfo(unitId string, vpns map[string]gin.H, useCache bool) (gin.H, error) { + unitFile, err := readUnitFile(unitId) + if err != nil { + return gin.H{}, err } - // define vpns object - var vpn gin.H + result := parseUnitFile(unitId, unitFile) - // loop through lines - for _, line := range lines { + // add info from unit + remote_info, err := GetUnitInfo(unitId, useCache) + if err == nil { + result["info"] = remote_info + } else { + result["info"] = gin.H{} + } - // get values from line - parts := strings.Split(line, "\t") + // add join code + result["join_code"] = utils.GetJoinCode(unitId) - // compose result - vpn = gin.H{ - "real_address": parts[2], - "virtual_address": parts[3], - "bytes_rcvd": parts[5], - "bytes_sent": parts[6], - "connected_since": parts[8], - } + // add vpn info + if vpns[unitId] != nil { + result["vpn"] = vpns[unitId] + } else { + result["vpn"] = gin.H{} } + return result, nil +} + +func readUnitFile(unitId string) ([]byte, error) { // read unit file unitFile, err := os.ReadFile(configuration.Config.OpenVPNCCDDir + "/" + unitId) // return results - return unitFile, vpn, err + return unitFile, err } -func parseUnitFile(unitId string, unitFile []byte, vpn gin.H) gin.H { +func parseUnitFile(unitId string, unitFile []byte) gin.H { // parse unit file parts := strings.Split(string(unitFile), "\n") parts = strings.Split(parts[0], " ") @@ -193,48 +158,36 @@ func parseUnitFile(unitId string, unitFile []byte, vpn gin.H) gin.H { "netmask": parts[2], } - // check if vpn data exists - if vpn != nil { - result["vpn"] = vpn - } else { - result["vpn"] = gin.H{} - } - - // retrieve unit info from database - info, err := storage.GetUnit(unitId) - if err == nil { - result["info"] = info - } else { - result["info"] = gin.H{} - } - - result["join_code"] = utils.GetJoinCode(unitId) return result } func GetUnit(c *gin.Context) { + // get cache query param + cache := c.DefaultQuery("cache", "true") + + // get vpn info + vpns := getVpnInfo() + // get unit id unitId := c.Param("unit_id") - // read unit file - unitFile, vpn, err := readUnitFile(unitId) + // parse unit file + result, err := getUnitInfo(unitId, vpns, cache == "true") + if err != nil { c.JSON(http.StatusBadRequest, structs.Map(response.StatusBadRequest{ Code: 400, - Message: "access CCD directory unit file failed", + Message: "Can't get unit info for: " + unitId, Data: err.Error(), })) + } else { + // return 200 OK with data + c.JSON(http.StatusOK, structs.Map(response.StatusOK{ + Code: 200, + Message: "unit listed successfully", + Data: result, + })) } - - // parse unit file - result := parseUnitFile(unitId, unitFile, vpn) - - // return 200 OK with data - c.JSON(http.StatusOK, structs.Map(response.StatusOK{ - Code: 200, - Message: "unit listed successfully", - Data: result, - })) } func GetToken(c *gin.Context) { @@ -489,7 +442,7 @@ func RegisterUnit(c *gin.Context) { credentials.Password = password } } else { - // update credentials + // create credentials credentials.Username = username credentials.Password = password } @@ -506,16 +459,6 @@ func RegisterUnit(c *gin.Context) { return } - errAdd := storage.AddOrUpdateUnit(jsonRequest.UnitId, jsonRequest.UnitName, jsonRequest.Version, jsonRequest.SubscriptionType, jsonRequest.SystemId) - if errAdd != nil { - c.JSON(http.StatusBadRequest, structs.Map(response.StatusBadRequest{ - Code: 400, - Message: "cannot add update to database for: " + jsonRequest.UnitId, - Data: errAdd.Error(), - })) - return - } - // return 200 OK with data c.JSON(http.StatusOK, structs.Map(response.StatusOK{ Code: 200, @@ -603,6 +546,44 @@ func DeleteUnit(c *gin.Context) { })) } +func GetUnitInfo(unitId string, useCache bool) (models.UnitInfo, error) { + + if useCache { + info, error := cache.GetUnitInfo(unitId) + if error == nil { + return info, nil + } + } + + // get remote info + unitInfo, err := GetRemoteInfo(unitId) + if err != nil { + return models.UnitInfo{}, err + } + + // cache is always updated + cache.SetUnitInfo(unitId, unitInfo) + + return unitInfo, nil +} + +// list unit name from files in OpenVPNCCDDir +func ListUnits() ([]string, error) { + units := []string{} + // list file in OpenVPNCCDDir + files, err := os.ReadDir(configuration.Config.OpenVPNCCDDir) + if err != nil { + return nil, err + } + + // loop through files + for _, file := range files { + units = append(units, file.Name()) + } + + return units, nil +} + func GetUnitToken(unitId string) (string, string, error) { // read credentials @@ -627,8 +608,8 @@ func GetUnitToken(unitId string) (string, string, error) { // set request header r.Header.Add("Content-Type", "application/json") - // make request - client := &http.Client{} + // make request, 2 seconds timeout + client := &http.Client{Timeout: 2 * time.Second} res, err := client.Do(r) if err != nil { return "", "", errors.New("request failed for: " + unitId) @@ -646,19 +627,13 @@ func GetUnitToken(unitId string) (string, string, error) { // check if token is not empty if len(loginResponse.Token) == 0 { - return "", "", errors.New("invalid JWT token response for: " + unitId) + return "", "", errors.New("invalid token response for: " + unitId) } return loginResponse.Token, loginResponse.Expire, nil } -func GetUnitInfo(unitId string) (models.UnitInfo, error) { - - info, error := cache.GetUnitInfo(unitId) - if error == nil { - return info, nil - } - +func GetRemoteInfo(unitId string) (models.UnitInfo, error) { // get the unit token and execute the request token, _, _ := GetUnitToken(unitId) if token == "" { @@ -690,8 +665,8 @@ func GetUnitInfo(unitId string) (models.UnitInfo, error) { r.Header.Add("Content-Type", "application/json") r.Header.Add("Authorization", "Bearer "+token) - // make request - client := &http.Client{} + // make request, with 2 seconds timeout + client := &http.Client{Timeout: 2 * time.Second} res, err := client.Do(r) if err != nil { return models.UnitInfo{}, errors.New("error making request") @@ -705,12 +680,5 @@ func GetUnitInfo(unitId string) (models.UnitInfo, error) { return models.UnitInfo{}, errors.New("error decoding response") } - // save to cache - // FIXME: add option to GET /units to ignore the cache - // FIXME: read all units on register - // FIXME: load unit info every hour - - cache.SetUnitInfo(unitId, unitInfo.Data) - return unitInfo.Data, nil } diff --git a/api/models/ubus.go b/api/models/ubus.go index 9460ce0..319d0c7 100644 --- a/api/models/ubus.go +++ b/api/models/ubus.go @@ -22,12 +22,3 @@ type UbusInfoResponse struct { Data UnitInfo `json:"data"` Message string `json:"message"` } - -type UnitInfo struct { - UnitName string `json:"unit_name"` - Version string `json:"version"` - SubscriptionType string `json:"subscription_type"` - SystemID string `json:"system_id"` - SSHPort int `json:"ssh_port"` - FQDN string `json:"fqdn"` -} diff --git a/api/models/unit.go b/api/models/unit.go index 182ac9e..f0e8765 100644 --- a/api/models/unit.go +++ b/api/models/unit.go @@ -9,7 +9,9 @@ package models -import "time" +import ( + "time" +) type AddRequest struct { UnitId string `json:"unit_id" binding:"required"` @@ -33,3 +35,12 @@ type Unit struct { SystemID string `json:"system_id" structs:"system_id"` Created time.Time `json:"created" structs:"created"` } + +type UnitInfo struct { + UnitName string `json:"unit_name"` + Version string `json:"version"` + SubscriptionType string `json:"subscription_type"` + SystemID string `json:"system_id"` + SSHPort int `json:"ssh_port"` + FQDN string `json:"fqdn"` +} diff --git a/api/storage/schema.sql b/api/storage/schema.sql index d5498fe..64a07d5 100644 --- a/api/storage/schema.sql +++ b/api/storage/schema.sql @@ -13,13 +13,4 @@ CREATE TABLE accounts ( `password` TEXT NOT NULL, `display_name` TEXT, `created` TIMESTAMP NOT NULL -); - -CREATE TABLE units ( - `id` TEXT NOT NULL PRIMARY KEY, - `name` TEXT NOT NULL UNIQUE, - `version` TEXT NOT NULL, - `system_id` TEXT, - `subscription_type` TEXT, - `created` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ); \ No newline at end of file diff --git a/api/storage/storage.go b/api/storage/storage.go index 7fb7f84..de1b5cc 100644 --- a/api/storage/storage.go +++ b/api/storage/storage.go @@ -262,77 +262,3 @@ func UpdatePassword(accountUsername string, newPassword string) error { return err } - -func AddOrUpdateUnit(unitID string, unitName string, version string, subscriptionType string, systemID string) error { - // get db - db := Instance() - - // define query - _, err := db.Exec( - "REPLACE INTO units (id, name, version, subscription_type, system_id) VALUES (?, ?, ?, ?, ?)", - unitID, - unitName, - version, - subscriptionType, - systemID, - ) - - // check error - if err != nil { - logs.Logs.Println("[ERR][STORAGE][ADD_OR_UPDATE_UNIT] error in insert units query: " + err.Error()) - } - - return err -} - -func GetUnit(unitId string) (models.Unit, error) { - // get db - db := Instance() - - // define query - query := "SELECT id, name, version, subscription_type, system_id, created FROM units where id = ?" - rows, err := db.Query(query, unitId) - if err != nil { - logs.Logs.Println("[ERR][STORAGE][GET_UNIT] error in query execution:" + err.Error()) - } - defer rows.Close() - - // loop rows - var result models.Unit - for rows.Next() { - if err := rows.Scan(&result.ID, &result.Name, &result.Version, &result.SubscriptionType, &result.SystemID, &result.Created); err != nil { - logs.Logs.Println("[ERR][STORAGE][GET_UNIT] error in query row extraction" + err.Error()) - } - } - - // return results - return result, err -} - -func GetUnits() ([]models.Unit, error) { - // get db - db := Instance() - - // define query - query := "SELECT id, name, version, subscription_type, system_id, created FROM units" - rows, err := db.Query(query) - if err != nil { - logs.Logs.Println("[ERR][STORAGE][GET_UNITS] error in query execution:" + err.Error()) - } - defer rows.Close() - - // loop rows - var results []models.Unit - for rows.Next() { - var unitRow models.Unit - if err := rows.Scan(&unitRow.ID, &unitRow.Name, &unitRow.Version, &unitRow.SubscriptionType, &unitRow.SystemID, &unitRow.Created); err != nil { - logs.Logs.Println("[ERR][STORAGE][GET_UNITS] error in query row extraction" + err.Error()) - } - - // append results - results = append(results, unitRow) - } - - // return results - return results, err -}