From eb0aeae6cd8d0c0cac3fb29294f852373224be79 Mon Sep 17 00:00:00 2001 From: cx-henriqueAlvelos Date: Wed, 5 Jul 2023 16:02:21 +0100 Subject: [PATCH 1/8] added linesResolved counter --- pkg/kics/sink.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/pkg/kics/sink.go b/pkg/kics/sink.go index 8d9ba7ddeb8..95cb572ed1d 100644 --- a/pkg/kics/sink.go +++ b/pkg/kics/sink.go @@ -47,6 +47,15 @@ func (s *Service) sink(ctx context.Context, filename, scanID string, rc io.Reade return nil } + linesResolved := 0 + resolvedFiles := make(map[string]bool) + for _, ref := range documents.ResolvedFiles { + if _, exists := resolvedFiles[ref.Path]; !exists && ref.Path != filename { + linesResolved += len(*ref.LinesContent) + } + } + s.Tracker.TrackFileFoundCountLines(linesResolved) + fileCommands := s.Parser.CommentsCommands(filename, *content) for _, document := range documents.Docs { @@ -85,7 +94,7 @@ func (s *Service) sink(ctx context.Context, filename, scanID string, rc io.Reade s.Tracker.TrackFileParse() log.Debug().Msgf("Finished to process file %s", filename) - s.Tracker.TrackFileParseCountLines(documents.CountLines - len(documents.IgnoreLines)) + s.Tracker.TrackFileParseCountLines(documents.CountLines + linesResolved - len(documents.IgnoreLines)) s.Tracker.TrackFileIgnoreCountLines(len(documents.IgnoreLines)) return errors.Wrap(err, "failed to save file content") From 48c8e7733634bdd0b5d4f80e27e97e7f4789f612 Mon Sep 17 00:00:00 2001 From: cx-henriqueAlvelos Date: Wed, 5 Jul 2023 18:05:45 +0100 Subject: [PATCH 2/8] fix for resolvedFiles store files instead of slices --- pkg/kics/sink.go | 3 +-- pkg/resolver/file/file.go | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/pkg/kics/sink.go b/pkg/kics/sink.go index 95cb572ed1d..29f715b554d 100644 --- a/pkg/kics/sink.go +++ b/pkg/kics/sink.go @@ -48,9 +48,8 @@ func (s *Service) sink(ctx context.Context, filename, scanID string, rc io.Reade } linesResolved := 0 - resolvedFiles := make(map[string]bool) for _, ref := range documents.ResolvedFiles { - if _, exists := resolvedFiles[ref.Path]; !exists && ref.Path != filename { + if ref.Path != filename { linesResolved += len(*ref.LinesContent) } } diff --git a/pkg/resolver/file/file.go b/pkg/resolver/file/file.go index f7e9d3aa7dd..43ce6eebe15 100644 --- a/pkg/resolver/file/file.go +++ b/pkg/resolver/file/file.go @@ -247,7 +247,7 @@ func (r *Resolver) resolveYamlPath( } } - r.ResolvedFiles[value] = model.ResolvedFile{ + r.ResolvedFiles[filename] = model.ResolvedFile{ Content: resolvedFilesCache[filename].fileContent, Path: path, LinesContent: utils.SplitLines(string(resolvedFilesCache[filename].fileContent)), @@ -367,7 +367,7 @@ func (r *Resolver) resolvePath( } } - r.ResolvedFiles[value] = model.ResolvedFile{ + r.ResolvedFiles[onlyFilePath] = model.ResolvedFile{ Content: resolvedFilesCache[onlyFilePath].fileContent, Path: path, LinesContent: utils.SplitLines(string(resolvedFilesCache[onlyFilePath].fileContent)), From 08cb33fc23103e2e4f62c773b08c908b0b66b715 Mon Sep 17 00:00:00 2001 From: cx-henriqueAlvelos Date: Thu, 6 Jul 2023 12:00:43 +0100 Subject: [PATCH 3/8] changed file --- pkg/resolver/file/file.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/pkg/resolver/file/file.go b/pkg/resolver/file/file.go index 43ce6eebe15..14bf5caeb00 100644 --- a/pkg/resolver/file/file.go +++ b/pkg/resolver/file/file.go @@ -247,7 +247,7 @@ func (r *Resolver) resolveYamlPath( } } - r.ResolvedFiles[filename] = model.ResolvedFile{ + r.ResolvedFiles[getPathFromString(value)] = model.ResolvedFile{ Content: resolvedFilesCache[filename].fileContent, Path: path, LinesContent: utils.SplitLines(string(resolvedFilesCache[filename].fileContent)), @@ -330,6 +330,14 @@ func (r *Resolver) resolveFile( return nil, false } +func getPathFromString(path string) string { + lastIndex := strings.LastIndex(path, "#") + if lastIndex == -1 { + return path + } + return path[:lastIndex] +} + // isPath returns true if the value is a valid path func (r *Resolver) resolvePath( originalFileContent []byte, @@ -367,7 +375,7 @@ func (r *Resolver) resolvePath( } } - r.ResolvedFiles[onlyFilePath] = model.ResolvedFile{ + r.ResolvedFiles[getPathFromString(value)] = model.ResolvedFile{ Content: resolvedFilesCache[onlyFilePath].fileContent, Path: path, LinesContent: utils.SplitLines(string(resolvedFilesCache[onlyFilePath].fileContent)), From a82d3153ff65f7daa51d26ac732ef26cafc4ca90 Mon Sep 17 00:00:00 2001 From: cx-henriqueAlvelos <128044259+cx-henriqueAlvelos@users.noreply.github.com> Date: Wed, 12 Jul 2023 10:05:45 +0000 Subject: [PATCH 4/8] changed onlyfilePath --- pkg/resolver/file/file.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pkg/resolver/file/file.go b/pkg/resolver/file/file.go index 14bf5caeb00..801bfc6ca61 100644 --- a/pkg/resolver/file/file.go +++ b/pkg/resolver/file/file.go @@ -226,9 +226,7 @@ func (r *Resolver) resolveYamlPath( value = checkServerlessFileReference(value) path := filepath.Join(filepath.Dir(filePath), value) - splitPath = strings.Split(path, "#") // splitting by removing the section to look for in the file - // index 0 contains the path of the file while the other indexes contain the sections (e.g. path = "./definitions.json#User/schema") - onlyFilePath := splitPath[0] + onlyFilePath := getPathFromString(path) _, err := os.Stat(path) if err != nil { return *v, false @@ -259,7 +257,7 @@ func (r *Resolver) resolveYamlPath( if strings.Contains(strings.ToLower(value), "!ref") { // Cloudformation !Ref check return *obj, false } - if len(splitPath) == 1 { + if !strings.Contains(path, "#") { return *obj, true } } From f47b5fe39903a8e3ef02deb5f6977bcf27fc25b0 Mon Sep 17 00:00:00 2001 From: cx-henriqueAlvelos Date: Wed, 12 Jul 2023 16:39:43 +0100 Subject: [PATCH 5/8] Added Indentation to Resolved Files --- pkg/resolver/file/file.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/resolver/file/file.go b/pkg/resolver/file/file.go index 14bf5caeb00..81533530399 100644 --- a/pkg/resolver/file/file.go +++ b/pkg/resolver/file/file.go @@ -1,6 +1,7 @@ package file import ( + "encoding/json" "errors" "io" "os" @@ -58,7 +59,7 @@ func (r *Resolver) Resolve(fileContent []byte, path string, resolveCount int, re // resolve the paths obj, _ = r.walk(fileContent, obj, obj, path, resolveCount, resolvedFilesCache) - b, err := r.marshler(obj) + b, err := json.MarshalIndent(obj, "", "") if err != nil { return fileContent } From cac2831b2fab8e9a0f7f6b3e0932e0f26e485548 Mon Sep 17 00:00:00 2001 From: cx-henriqueAlvelos Date: Wed, 19 Jul 2023 12:38:25 +0100 Subject: [PATCH 6/8] Parsed lines count changed --- pkg/kics/sink.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/kics/sink.go b/pkg/kics/sink.go index 29f715b554d..a1acf322b32 100644 --- a/pkg/kics/sink.go +++ b/pkg/kics/sink.go @@ -93,7 +93,7 @@ func (s *Service) sink(ctx context.Context, filename, scanID string, rc io.Reade s.Tracker.TrackFileParse() log.Debug().Msgf("Finished to process file %s", filename) - s.Tracker.TrackFileParseCountLines(documents.CountLines + linesResolved - len(documents.IgnoreLines)) + s.Tracker.TrackFileParseCountLines(documents.CountLines - len(documents.IgnoreLines)) s.Tracker.TrackFileIgnoreCountLines(len(documents.IgnoreLines)) return errors.Wrap(err, "failed to save file content") From 653721cc08ddddfe94f52b9e51268104256ba783 Mon Sep 17 00:00:00 2001 From: cx-henriqueAlvelos Date: Wed, 19 Jul 2023 15:55:08 +0100 Subject: [PATCH 7/8] dummy commit --- pkg/kics/sink.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/kics/sink.go b/pkg/kics/sink.go index a1acf322b32..9636117a5dd 100644 --- a/pkg/kics/sink.go +++ b/pkg/kics/sink.go @@ -91,9 +91,11 @@ func (s *Service) sink(ctx context.Context, filename, scanID string, rc io.Reade s.saveToFile(ctx, &file) } s.Tracker.TrackFileParse() + log.Debug().Msgf("Finished to process file %s", filename) s.Tracker.TrackFileParseCountLines(documents.CountLines - len(documents.IgnoreLines)) + s.Tracker.TrackFileIgnoreCountLines(len(documents.IgnoreLines)) return errors.Wrap(err, "failed to save file content") From 8f07e8ed6cdc5e214bfc02069759bb77821e7c59 Mon Sep 17 00:00:00 2001 From: cx-henriqueAlvelos Date: Wed, 19 Jul 2023 15:55:51 +0100 Subject: [PATCH 8/8] fix commit --- pkg/kics/sink.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/kics/sink.go b/pkg/kics/sink.go index 9636117a5dd..a1acf322b32 100644 --- a/pkg/kics/sink.go +++ b/pkg/kics/sink.go @@ -91,11 +91,9 @@ func (s *Service) sink(ctx context.Context, filename, scanID string, rc io.Reade s.saveToFile(ctx, &file) } s.Tracker.TrackFileParse() - log.Debug().Msgf("Finished to process file %s", filename) s.Tracker.TrackFileParseCountLines(documents.CountLines - len(documents.IgnoreLines)) - s.Tracker.TrackFileIgnoreCountLines(len(documents.IgnoreLines)) return errors.Wrap(err, "failed to save file content")