Skip to content

Commit

Permalink
feat(file-with-console): updated code based to broadcast and write it… (
Browse files Browse the repository at this point in the history
#60)

* feat(file-with-console): updated code base to broadcast and write it to file in JSON format as well as console
  • Loading branch information
utsavmaniyar authored Jun 1, 2023
1 parent 168808c commit 92e6123
Show file tree
Hide file tree
Showing 8 changed files with 179 additions and 32 deletions.
4 changes: 3 additions & 1 deletion docs/USAGE.md
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ Usage of go-earlybird:
-file string
Output file -- e.g., 'go-earlybird --file=/home/jdoe/myfile.csv'
-format string
Output format [ console | json | csv ] (default "console")
Output format [ console | json | csv ] (default "console").
-git string
Full URL to a git repo to scan e.g. github.com/user/repo
-git-branch string
Expand Down Expand Up @@ -97,6 +97,8 @@ Usage of go-earlybird:
Update module configurations
-verbose
Reports details about file reads
-with-console
Prints findings in console with JSON format report
-workers int
Set number of workers. (default 100)
-worksize int
Expand Down
3 changes: 1 addition & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -42,9 +42,8 @@ require (
github.com/src-d/gcfg v1.4.0 // indirect
github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf // indirect
github.com/xanzy/ssh-agent v0.2.1 // indirect
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect
golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 // indirect
golang.org/x/sys v0.0.0-20220906165534-d0df966e6959 // indirect
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
google.golang.org/protobuf v1.23.0 // indirect
gopkg.in/src-d/go-billy.v4 v4.3.2 // indirect
gopkg.in/warnings.v0 v0.1.2 // indirect
Expand Down
106 changes: 106 additions & 0 deletions pkg/broadcast/broadcast.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,106 @@
/*
* Copyright 2023 American Express
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied. See the License for the specific language governing
* permissions and limitations under the License.
*/

/*
* Broadcast package is used to create multiple channel reader.
* Since once we read data from channel, it empties out it self.
* this package provides a way to broadcast the data and assign listern to that data.
*/
package broadcast

import (
"context"
"github.com/americanexpress/earlybird/pkg/scan"
)

type BroadcastServer interface {
Subscribe() <-chan scan.Hit
CancelSubscription(<-chan scan.Hit)
}

type broadcastServer struct {
source <-chan scan.Hit
listeners []chan scan.Hit
addListener chan chan scan.Hit
removeListener chan (<-chan scan.Hit)
}

// Subscribe() creates a subcribtion on broadcastServer.
func (s *broadcastServer) Subscribe() <-chan scan.Hit {
newListener := make(chan scan.Hit)
s.addListener <- newListener
return newListener
}

// CancelSubscription() cancel a subcribtion on broadcastServer.
func (s *broadcastServer) CancelSubscription(channel <-chan scan.Hit) {
s.removeListener <- channel
}

// NewBroadcastServer() create a broadcast server and starts new routine.
func NewBroadcastServer(ctx context.Context, source <-chan scan.Hit) BroadcastServer {
service := &broadcastServer{
source: source,
listeners: make([]chan scan.Hit, 0),
addListener: make(chan chan scan.Hit),
removeListener: make(chan (<-chan scan.Hit)),
}
go service.serve(ctx)
return service
}

// serve() run the server and manages listener counts.
func (s *broadcastServer) serve(ctx context.Context) {
defer func() {
for _, listener := range s.listeners {
if listener != nil {
close(listener)
}
}
}()

for {
select {
case <-ctx.Done():
return
case newListener := <-s.addListener:
s.listeners = append(s.listeners, newListener)
case listenerToRemove := <-s.removeListener:
for i, ch := range s.listeners {
if ch == listenerToRemove {
s.listeners[i] = s.listeners[len(s.listeners)-1]
s.listeners = s.listeners[:len(s.listeners)-1]
close(ch)
break
}
}
case val, ok := <-s.source:
if !ok {
return
}
for _, listener := range s.listeners {
if listener != nil {
select {
case listener <- val:
case <-ctx.Done():
return
}

}
}
}
}
}
1 change: 1 addition & 0 deletions pkg/config/structures.go
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@ type EarlybirdConfig struct {
TargetType string
EnabledModulesMap map[string]string
EnabledModules []string
WithConsole bool
OutputFormat string
OutputFile string
IgnoreFile string
Expand Down
1 change: 1 addition & 0 deletions pkg/core/const.go
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@ var (
ptrGitTrackedFlag = flag.Bool("git-tracked", false, "Scan only git tracked files")
ptrPath = flag.String("path", utils.MustGetWD(), "Directory to scan (defaults to CWD) -- ABSOLUTE PATH ONLY")
ptrOutputFormat = flag.String("format", "console", "Output format [ console | json | csv ]")
ptrWithConsole = flag.Bool("with-console", false, "While using --format, this flag will help to print findings in console")
ptrOutputFile = flag.String("file", "", "Output file -- e.g., 'go-earlybird --file=/home/jdoe/myfile.csv'")
ptrIgnoreFile = flag.String("ignorefile", userHomeDir+string(os.PathSeparator)+".ge_ignore", "Patterns File (including wildcards) for files to ignore. (e.g. *.jpg)")
ptrFailSeverityThreshold = flag.String("fail-severity", cfgreader.Settings.TranslateLevelID(cfgreader.Settings.FailThreshold), "Lowest severity level at which to fail "+levelOptions)
Expand Down
59 changes: 33 additions & 26 deletions pkg/core/core.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,15 +18,18 @@ package core

import (
"bufio"
"context"
"flag"
"fmt"
"github.com/americanexpress/earlybird/pkg/broadcast"
"github.com/americanexpress/earlybird/pkg/buildflags"
"log"
"net/http"
"os"
"path"
"path/filepath"
"strings"
"sync"
"time"

"github.com/americanexpress/earlybird/pkg/api"
Expand Down Expand Up @@ -193,6 +196,7 @@ func (eb *EarlybirdCfg) ConfigInit() {
eb.Config.VerboseEnabled = *ptrVerbose
eb.Config.Suppress = *ptrSuppressSecret
eb.Config.OutputFormat = *ptrOutputFormat
eb.Config.WithConsole = *ptrWithConsole
eb.Config.OutputFile = *ptrOutputFile
eb.Config.SearchDir = *ptrPath
eb.Config.IgnoreFile = *ptrIgnoreFile
Expand Down Expand Up @@ -326,34 +330,37 @@ func (eb *EarlybirdCfg) FileContext() (fileContext file.Context, err error) {
func (eb *EarlybirdCfg) WriteResults(start time.Time, HitChannel chan scan.Hit, fileContext file.Context) {
// Send output to a writer
var err error
switch {
case eb.Config.OutputFormat == "json":
var Hits []scan.Hit
for hit := range HitChannel {
Hits = append(Hits, hit)
}

report := scan.Report{
Hits: Hits,
HitCount: len(Hits),
Skipped: fileContext.SkippedFiles,
Ignore: fileContext.IgnorePatterns,
Version: eb.Config.Version,
Modules: eb.Config.EnabledModules,
Threshold: eb.Config.SeverityDisplayLevel,
FilesScanned: len(fileContext.Files),
RulesObserved: len(scan.CombinedRules),
StartTime: start.UTC().Format(time.RFC3339),
EndTime: time.Now().UTC().Format(time.RFC3339),
Duration: fmt.Sprintf("%d ms", time.Since(start)/time.Millisecond),
if eb.Config.WithConsole && eb.Config.OutputFormat == "json" {
var wg sync.WaitGroup
wg.Add(2)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
broadcaster := broadcast.NewBroadcastServer(ctx, HitChannel)
listener1 := broadcaster.Subscribe()
listener2 := broadcaster.Subscribe()
go func() {
defer wg.Done()
err = writers.WriteConsole(listener1, "", eb.Config.ShowFullLine)
log.Printf("\n%d files scanned in %s", len(fileContext.Files), time.Since(start))
log.Printf("\n%d rules observed\n", len(scan.CombinedRules))
}()
go func() {
defer wg.Done()
err = writers.WriteJSON(listener2, eb.Config, fileContext, eb.Config.OutputFile)
}()
wg.Wait()
} else {
switch {
case eb.Config.OutputFormat == "json":
err = writers.WriteJSON(HitChannel, eb.Config, fileContext, eb.Config.OutputFile)
case eb.Config.OutputFormat == "csv":
err = writers.WriteCSV(HitChannel, eb.Config.OutputFile)
default:
err = writers.WriteConsole(HitChannel, eb.Config.OutputFile, eb.Config.ShowFullLine)
log.Printf("\n%d files scanned in %s", len(fileContext.Files), time.Since(start))
log.Printf("\n%d rules observed\n", len(scan.CombinedRules))
}
_, err = writers.WriteJSON(report, eb.Config.OutputFile)
case eb.Config.OutputFormat == "csv":
err = writers.WriteCSV(HitChannel, eb.Config.OutputFile)
default:
err = writers.WriteConsole(HitChannel, eb.Config.OutputFile, eb.Config.ShowFullLine)
log.Printf("\n%d files scanned in %s", len(fileContext.Files), time.Since(start))
log.Printf("\n%d rules observed\n", len(scan.CombinedRules))
}
if err != nil {
log.Println("Writing Results failed:", err)
Expand Down
35 changes: 33 additions & 2 deletions pkg/writers/jsonout.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,43 @@ package writers

import (
"encoding/json"
"fmt"
cfgReader "github.com/americanexpress/earlybird/pkg/config"
"github.com/americanexpress/earlybird/pkg/file"
"github.com/americanexpress/earlybird/pkg/scan"
"io/ioutil"
"os"
"time"
)

//WriteJSON Outputs an object as a JSON blob to an output file or console
func WriteJSON(v interface{}, fileName string) (s string, err error) {
// WriteJSON takes the hits, converts them into JSON report and passing report to reportToJSONWriter().
func WriteJSON(hits <-chan scan.Hit, config cfgReader.EarlybirdConfig, fileContext file.Context, fileName string) (err error) {
start := time.Now()
var Hits []scan.Hit
for hit := range hits {
Hits = append(Hits, hit)
}

report := scan.Report{
Hits: Hits,
HitCount: len(Hits),
Skipped: fileContext.SkippedFiles,
Ignore: fileContext.IgnorePatterns,
Version: config.Version,
Modules: config.EnabledModules,
Threshold: config.SeverityDisplayLevel,
FilesScanned: len(fileContext.Files),
RulesObserved: len(scan.CombinedRules),
StartTime: start.UTC().Format(time.RFC3339),
EndTime: time.Now().UTC().Format(time.RFC3339),
Duration: fmt.Sprintf("%d ms", time.Since(start)/time.Millisecond),
}
_, err = reportToJSONWriter(report, fileName)
return err
}

//reportToJSONWriter Outputs an object as a JSON blob to an output file or console
func reportToJSONWriter(v interface{}, fileName string) (s string, err error) {
b, err := json.MarshalIndent(v, "", "\t")
if err != nil {
return "", err
Expand Down
2 changes: 1 addition & 1 deletion pkg/writers/jsonout_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ func TestWriteJSON(t *testing.T) {
for _, myTest := range tests {
tt := myTest
t.Run(tt.name, func(t *testing.T) {
got, err := WriteJSON(tt.args.report, tt.args.fileName)
got, err := reportToJSONWriter(tt.args.report, tt.args.fileName)
if (err != nil) != tt.wantErr {
t.Errorf("WriteJSON() error = %v, wantErr %v", err, tt.wantErr)
return
Expand Down

0 comments on commit 92e6123

Please sign in to comment.