From 9ae0c4f9cd6d3aec92c5e782a0f404388553f7d9 Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Tue, 2 Jan 2024 20:38:50 +0530 Subject: [PATCH 1/3] add support for csv parsing Signed-off-by: MUzairS15 --- helpers/component_info.json | 2 +- models/meshmodel/core/v1alpha1/component.go | 2 +- utils/csv/csv.go | 98 +++++++++++++++++++++ utils/error.go | 5 ++ utils/kubernetes/crd.go | 12 +-- utils/utils.go | 4 +- 6 files changed, 113 insertions(+), 10 deletions(-) create mode 100644 utils/csv/csv.go diff --git a/helpers/component_info.json b/helpers/component_info.json index bd49f70e..ce350ff2 100644 --- a/helpers/component_info.json +++ b/helpers/component_info.json @@ -1,5 +1,5 @@ { "name": "meshkit", "type": "library", - "next_error_code": 11106 + "next_error_code": 11107 } \ No newline at end of file diff --git a/models/meshmodel/core/v1alpha1/component.go b/models/meshmodel/core/v1alpha1/component.go index 8137f635..af0546cf 100644 --- a/models/meshmodel/core/v1alpha1/component.go +++ b/models/meshmodel/core/v1alpha1/component.go @@ -177,7 +177,7 @@ func GetMeshModelComponents(db *database.Handler, f ComponentFilter) (c []Compon type ComponentFilter struct { Name string APIVersion string - Greedy bool //when set to true - instead of an exact match, name will be prefix matched + Greedy bool //when set to true - instead of an exact match, name will be matched as a substring Trim bool //when set to true - the schema is not returned DisplayName string ModelName string diff --git a/utils/csv/csv.go b/utils/csv/csv.go new file mode 100644 index 00000000..40020bba --- /dev/null +++ b/utils/csv/csv.go @@ -0,0 +1,98 @@ +package csv + +import ( + "context" + "encoding/csv" + "io" + "os" + "strings" + + "github.com/layer5io/meshkit/utils" +) + +type CSV[E any] struct { + Context context.Context + cancel context.CancelFunc + reader *csv.Reader + filePath string + lineForColNo int + // Stores the mapping for coumn name to golang equivalent attribute name. + // It is optional and default mapping is the lower case representation + // eg: ColumnnName: Descritption, equivalent golang attribute to which it will be mapped during unmarshal "description". + columnToNameMapping map[string]string + predicateFunc func(columns []string, currentRow []string) bool +} + +func NewCSVParser[E any](filePath string, lineForColNo int, colToNameMapping map[string]string, predicateFunc func(columns []string, currentRow []string) bool) (*CSV[E], error) { + reader, err := os.Open(filePath) + if err != nil { + return nil, utils.ErrReadFile(err, filePath) + } + + ctx, cancel := context.WithCancel(context.Background()) + return &CSV[E]{ + Context: ctx, + cancel: cancel, + reader: csv.NewReader(reader), + filePath: filePath, + columnToNameMapping: colToNameMapping, + lineForColNo: lineForColNo, + predicateFunc: predicateFunc, + }, nil +} + +// "lineForColNo" line number where the columns are defined in the csv +func (c *CSV[E]) ExtractCols(lineForColNo int) ([]string, error) { + data := []string{} + var err error + for i := 0; i <= lineForColNo; i++ { + data, err = c.reader.Read() + if err != nil { + return nil, utils.ErrReadFile(err, c.filePath) + } + } + return data, nil +} + +func (c *CSV[E]) Parse(ch chan E) error { + defer func() { + c.cancel() + }() + + columnNames, err := c.ExtractCols(c.lineForColNo) + size := len(columnNames) + if err != nil { + return utils.ErrReadFile(err, c.filePath) + } + for { + data := make(map[string]interface{}) + values, err := c.reader.Read() + if err == io.EOF { + break + } + + if err != nil { + return err + } + + if c.predicateFunc != nil && c.predicateFunc(columnNames, values) { + for index, value := range values { + if index < size { + attribute, ok := c.columnToNameMapping[columnNames[index]] + if !ok { + attribute = strings.ReplaceAll(strings.ToLower(columnNames[index]), " ", "_") + } + data[attribute] = value + } + } + + parsedData, err := utils.MarshalAndUnmarshal[map[string]interface{}, E](data) + if err != nil { + return utils.ErrReadFile(err, c.filePath) + } + ch <- parsedData + } + + } + return nil +} diff --git a/utils/error.go b/utils/error.go index 55408b4e..08be9027 100644 --- a/utils/error.go +++ b/utils/error.go @@ -21,6 +21,7 @@ var ( ErrRemoteFileNotFoundCode = "11052" ErrReadingRemoteFileCode = "11053" ErrReadingLocalFileCode = "11054" + ErrReadFileCode = "11106" ErrGettingLatestReleaseTagCode = "11055" ErrInvalidProtocol = errors.New(ErrInvalidProtocolCode, errors.Alert, []string{"invalid protocol: only http, https and file are valid protocols"}, []string{}, []string{"Network protocol is incorrect"}, []string{"Make sure to specify the right network protocol"}) ErrMissingFieldCode = "11076" @@ -100,6 +101,10 @@ func ErrReadingLocalFile(err error) error { return errors.New(ErrReadingLocalFileCode, errors.Alert, []string{"error reading local file"}, []string{err.Error()}, []string{"File does not exist in the location (~/.kube/config)", "File is absent. Filename is not 'config'.", "Insufficient permissions to read file"}, []string{"Verify that the available kubeconfig is accessible by Meshery Server - verify sufficient file permissions (only needs read permission)."}) } +func ErrReadFile(err error, filepath string) error { + return errors.New(ErrReadFileCode, errors.Alert, []string{"error reading file"}, []string{err.Error()}, []string{fmt.Sprintf("File does not exist in the location %s", filepath), "Insufficient permissions"}, []string{"Verify that file exist at the provided location", "Verify sufficient file permissions."}) +} + func ErrGettingLatestReleaseTag(err error) error { return errors.New( ErrGettingLatestReleaseTagCode, diff --git a/utils/kubernetes/crd.go b/utils/kubernetes/crd.go index 05b6f6ee..c51c177e 100644 --- a/utils/kubernetes/crd.go +++ b/utils/kubernetes/crd.go @@ -17,9 +17,9 @@ type CRDItem struct { } type Spec struct { - Names names `json:"names"` - Group string `json:"group"` - Versions []struct{ + Names names `json:"names"` + Group string `json:"group"` + Versions []struct { Name string `json:"name"` } `json:"versions"` } @@ -47,8 +47,8 @@ func GetAllCustomResourcesInCluster(ctx context.Context, client rest.Interface) func GetGVRForCustomResources(crd *CRDItem) *schema.GroupVersionResource { return &schema.GroupVersionResource{ - Group: crd.Spec.Group, - Version: crd.Spec.Versions[0].Name, + Group: crd.Spec.Group, + Version: crd.Spec.Versions[0].Name, Resource: crd.Spec.Names.ResourceName, } -} \ No newline at end of file +} diff --git a/utils/utils.go b/utils/utils.go index 6767a1ca..e6a6a678 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -270,9 +270,9 @@ func MarshalAndUnmarshal[fromType any, toType any](val fromType) (unmarshalledva func IsClosed[K any](ch chan K) bool { select { - case <- ch: + case <-ch: return true default: return false } -} \ No newline at end of file +} From d011156f01b7415a924f43ab0fe21dcd758d1181 Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Tue, 2 Jan 2024 20:40:03 +0530 Subject: [PATCH 2/3] update comment Signed-off-by: MUzairS15 --- utils/csv/csv.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/csv/csv.go b/utils/csv/csv.go index 40020bba..fc47da91 100644 --- a/utils/csv/csv.go +++ b/utils/csv/csv.go @@ -17,7 +17,7 @@ type CSV[E any] struct { filePath string lineForColNo int // Stores the mapping for coumn name to golang equivalent attribute name. - // It is optional and default mapping is the lower case representation + // It is optional and default mapping is the lower case representation with spaces replaced with "_" // eg: ColumnnName: Descritption, equivalent golang attribute to which it will be mapped during unmarshal "description". columnToNameMapping map[string]string predicateFunc func(columns []string, currentRow []string) bool From dfda4118a8a506863100beb79b1c4648875ac653 Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Tue, 2 Jan 2024 20:51:38 +0530 Subject: [PATCH 3/3] add nil check Signed-off-by: MUzairS15 --- utils/csv/csv.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/utils/csv/csv.go b/utils/csv/csv.go index fc47da91..5a450d9f 100644 --- a/utils/csv/csv.go +++ b/utils/csv/csv.go @@ -78,11 +78,13 @@ func (c *CSV[E]) Parse(ch chan E) error { if c.predicateFunc != nil && c.predicateFunc(columnNames, values) { for index, value := range values { if index < size { - attribute, ok := c.columnToNameMapping[columnNames[index]] - if !ok { - attribute = strings.ReplaceAll(strings.ToLower(columnNames[index]), " ", "_") + if c.columnToNameMapping != nil { + attribute, ok := c.columnToNameMapping[columnNames[index]] + if !ok { + attribute = strings.ReplaceAll(strings.ToLower(columnNames[index]), " ", "_") + } + data[attribute] = value } - data[attribute] = value } }