From 1bbe07c2055e61d6d34d0f4acb8bd9194358fcaf Mon Sep 17 00:00:00 2001 From: Matt Cadorette Date: Fri, 4 Jun 2021 11:50:25 -0400 Subject: [PATCH] feat(cli): Add CSV rendering output for select commands (#424) Adds the ability to get CSV output from three commands using the `--csv` flag: * `lacework comp [csp] get-report` * `lacework vuln host list-cves` * `lacework vuln host list-hosts` JIRA: ALLY-465 --- cli/cmd/cli_state.go | 18 +++++++-- cli/cmd/compliance.go | 3 ++ cli/cmd/compliance_aws.go | 15 ++++++++ cli/cmd/compliance_azure.go | 16 ++++++++ cli/cmd/compliance_gcp.go | 16 ++++++++ cli/cmd/outputs.go | 44 ++++++++++++++++++++++ cli/cmd/outputs_test.go | 75 +++++++++++++++++++++++++++++++++++++ cli/cmd/vuln_host.go | 49 ++++++++++++++++++++++-- cli/cmd/vulnerability.go | 13 +++++++ 9 files changed, 242 insertions(+), 7 deletions(-) create mode 100644 cli/cmd/outputs_test.go diff --git a/cli/cmd/cli_state.go b/cli/cmd/cli_state.go index a6b205620..8cfa045ef 100644 --- a/cli/cmd/cli_state.go +++ b/cli/cmd/cli_state.go @@ -60,6 +60,7 @@ type cliState struct { workers sync.WaitGroup spinner *spinner.Spinner jsonOutput bool + csvOutput bool nonInteractive bool profileDetails map[string]interface{} } @@ -210,7 +211,7 @@ func (c *cliState) NewClient() error { // InteractiveMode returns true if the cli is running in interactive mode func (c *cliState) InteractiveMode() bool { - return !c.nonInteractive + return !c.nonInteractive && !c.csvOutput } // NonInteractive turns off interactive mode, that is, no progress bars and spinners @@ -274,14 +275,25 @@ func (c *cliState) EnableHumanOutput() { c.jsonOutput = false } +// EnableCSVOutput enables the cli to display CSV output +func (c *cliState) EnableCSVOutput() { + c.Log.Info("switch output to csv format") + c.csvOutput = true +} + // JSONOutput returns true if the cli is configured to display JSON output func (c *cliState) JSONOutput() bool { return c.jsonOutput } -// HumanOutput returns true if the cli is configured to siplay human readable output +// HumanOutput returns true if the cli is configured to display human readable output func (c *cliState) HumanOutput() bool { - return !c.jsonOutput + return !c.jsonOutput && !c.csvOutput +} + +// CSVOutput returns true if the cli is configured to display csv output +func (c *cliState) CSVOutput() bool { + return c.csvOutput } // loadStateFromViper loads parameters and environment variables diff --git a/cli/cmd/compliance.go b/cli/cmd/compliance.go index 911b9b660..b3cf1c197 100644 --- a/cli/cmd/compliance.go +++ b/cli/cmd/compliance.go @@ -39,6 +39,9 @@ var ( // download report in PDF format Pdf bool + // output report in CSV format + Csv bool + // display extended details about a compliance report Details bool diff --git a/cli/cmd/compliance_aws.go b/cli/cmd/compliance_aws.go index a20f6fb56..4f78c54a1 100644 --- a/cli/cmd/compliance_aws.go +++ b/cli/cmd/compliance_aws.go @@ -84,6 +84,10 @@ Then navigate to Settings > Integrations > Cloud Accounts. Use: "get-report ", Aliases: []string{"get"}, PreRunE: func(_ *cobra.Command, _ []string) error { + if compCmdState.Csv { + cli.EnableCSVOutput() + } + switch compCmdState.Type { case "CIS": compCmdState.Type = fmt.Sprintf("AWS_%s_S3", compCmdState.Type) @@ -176,6 +180,13 @@ To run an ad-hoc compliance assessment of an AWS account: } recommendations := complianceReportRecommendationsTable(report.Recommendations) + if cli.CSVOutput() { + return cli.OutputCSV( + []string{"ID", "Recommendation", "Status", "Severity", "Service", "Affected", "Assessed"}, + recommendations, + ) + } + cli.OutputHuman("\n") cli.OutputHuman( buildComplianceReportTable( @@ -236,6 +247,10 @@ func init() { "download report in PDF format", ) + complianceAwsGetReportCmd.Flags().BoolVar(&compCmdState.Csv, "csv", false, + "output report in CSV format", + ) + // AWS report types: AWS_CIS_S3, NIST_800-53_Rev4, ISO_2700, HIPAA, SOC, or PCI complianceAwsGetReportCmd.Flags().StringVar(&compCmdState.Type, "type", "CIS", "report type to display, supported types: CIS, NIST_800-53_Rev4, ISO_2700, HIPAA, SOC, or PCI", diff --git a/cli/cmd/compliance_azure.go b/cli/cmd/compliance_azure.go index 624d8e7ab..88b169edf 100644 --- a/cli/cmd/compliance_azure.go +++ b/cli/cmd/compliance_azure.go @@ -128,6 +128,10 @@ Then navigate to Settings > Integrations > Cloud Accounts. Use: "get-report ", Aliases: []string{"get"}, PreRunE: func(_ *cobra.Command, _ []string) error { + if compCmdState.Csv { + cli.EnableCSVOutput() + } + switch compCmdState.Type { case "CIS", "SOC", "PCI": compCmdState.Type = fmt.Sprintf("AZURE_%s", compCmdState.Type) @@ -218,6 +222,13 @@ To run an ad-hoc compliance assessment use the command: } recommendations := complianceReportRecommendationsTable(report.Recommendations) + if cli.CSVOutput() { + return cli.OutputCSV( + []string{"ID", "Recommendation", "Status", "Severity", "Service", "Affected", "Assessed"}, + recommendations, + ) + } + cli.OutputHuman("\n") cli.OutputHuman( buildComplianceReportTable( @@ -275,6 +286,11 @@ func init() { "download report in PDF format", ) + // Output the report in CSV format + complianceAzureGetReportCmd.Flags().BoolVar(&compCmdState.Csv, "csv", false, + "output report in CSV format", + ) + // Azure report types: AZURE_CIS, AZURE_SOC, or AZURE_PCI complianceAzureGetReportCmd.Flags().StringVar(&compCmdState.Type, "type", "CIS", "report type to display, supported types: CIS, SOC, or PCI", diff --git a/cli/cmd/compliance_gcp.go b/cli/cmd/compliance_gcp.go index c668cc3ce..7ffcde9f9 100644 --- a/cli/cmd/compliance_gcp.go +++ b/cli/cmd/compliance_gcp.go @@ -83,6 +83,10 @@ Then, select one GUID from an integration and visualize its details using the co Use: "get-report ", Aliases: []string{"get"}, PreRunE: func(_ *cobra.Command, _ []string) error { + if compCmdState.Csv { + cli.EnableCSVOutput() + } + switch compCmdState.Type { case "CIS", "SOC", "PCI": compCmdState.Type = fmt.Sprintf("GCP_%s", compCmdState.Type) @@ -173,6 +177,13 @@ To run an ad-hoc compliance assessment use the command: } recommendations := complianceReportRecommendationsTable(report.Recommendations) + if cli.CSVOutput() { + return cli.OutputCSV( + []string{"ID", "Recommendation", "Status", "Severity", "Service", "Affected", "Assessed"}, + recommendations, + ) + } + cli.OutputHuman("\n") cli.OutputHuman( buildComplianceReportTable( @@ -229,6 +240,11 @@ func init() { "download report in PDF format", ) + // Output the report in CSV format + complianceGcpGetReportCmd.Flags().BoolVar(&compCmdState.Csv, "csv", false, + "output report in CSV format", + ) + // GCP report types: GCP_CIS, GCP_SOC, or GCP_PCI. complianceGcpGetReportCmd.Flags().StringVar(&compCmdState.Type, "type", "CIS", "report type to display, supported types: CIS, SOC, or PCI", diff --git a/cli/cmd/outputs.go b/cli/cmd/outputs.go index 3210f8363..fe94594eb 100644 --- a/cli/cmd/outputs.go +++ b/cli/cmd/outputs.go @@ -19,11 +19,13 @@ package cmd import ( + "encoding/csv" "fmt" "os" "strings" "github.com/fatih/color" + "github.com/pkg/errors" ) // OutputJSON will print out the JSON representation of the provided data @@ -64,3 +66,45 @@ func (c *cliState) FormatJSONString(s string) (string, error) { } return string(pretty), nil } + +// Used to clean CSV inputs prior to rendering +func csvCleanData(input []string) []string { + var data []string + for _, h := range input { + data = append(data, strings.Replace(h, "\n", "", -1)) + } + return data +} + +// Used to produce CSV output +func renderAsCSV(headers []string, data [][]string) (string, error) { + csvOut := &strings.Builder{} + csv := csv.NewWriter(csvOut) + + if len(headers) > 0 { + if err := csv.Write(csvCleanData(headers)); err != nil { + return "", errors.Wrap(err, "Failed to build csv output") + } + } + + for _, record := range data { + if err := csv.Write(csvCleanData(record)); err != nil { + return "", errors.Wrap(err, "Failed to build csv output") + } + } + + // Write any buffered data to the underlying writer (standard output). + csv.Flush() + return csvOut.String(), csv.Error() +} + +// OutputCSV will print out the provided headers/data in CSV format +func (c *cliState) OutputCSV(headers []string, data [][]string) error { + csv, err := renderAsCSV(headers, data) + if err != nil { + return err + } + + fmt.Fprint(os.Stdout, csv) + return nil +} diff --git a/cli/cmd/outputs_test.go b/cli/cmd/outputs_test.go new file mode 100644 index 000000000..274c13554 --- /dev/null +++ b/cli/cmd/outputs_test.go @@ -0,0 +1,75 @@ +// +// Author:: Matt Cadorette () +// Copyright:: Copyright 2020, Lacework Inc. +// License:: Apache License, Version 2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package cmd + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestRenderSimpleCSV(t *testing.T) { + expectedCsv := strings.TrimPrefix(` +KEY,VALUE +key1,value1 +key2,value2 +key3,value3 +`, "\n") + + csv, _ := renderAsCSV( + []string{"KEY", "VALUE"}, + [][]string{ + {"key1", "value1"}, + {"key2", "value2"}, + {"key3", "value3"}, + }, + ) + assert.Equal(t, + csv, + expectedCsv, + "csv is not being formatted correctly") +} + +func TestRenderComplexCSV(t *testing.T) { + expectedCsv := strings.TrimPrefix(` +KEY HEADER VALUE,"VALUE,TEST" +key1,"this is a value, from [a, b, c]" +key2,value2 +key3,value3 +`, "\n") + + csv, _ := renderAsCSV( + []string{"KEY\n HEADER VALUE", "VALUE,TEST"}, + [][]string{ + {"key1", "this is a value, from [a, b, c]"}, + {"key2", "value2"}, + {"key3", "value3"}, + }, + ) + assert.Equal(t, + csv, + expectedCsv, + "csv is not being formatted correctly") +} + +func TestCSVDataCleanup(t *testing.T) { + data := csvCleanData([]string{"KEY\n HEADER\n VALUE", "VALUE,TEST\n"}) + assert.NotContains(t, strings.Join(data, ""), "\n", "data is not being cleaned up properly") +} diff --git a/cli/cmd/vuln_host.go b/cli/cmd/vuln_host.go index ced3e587d..8b717a999 100644 --- a/cli/cmd/vuln_host.go +++ b/cli/cmd/vuln_host.go @@ -187,8 +187,14 @@ To generate a package-manifest from the local host and scan it automatically: } vulHostListCvesCmd = &cobra.Command{ - Use: "list-cves", - Args: cobra.NoArgs, + Use: "list-cves", + Args: cobra.NoArgs, + PreRunE: func(_ *cobra.Command, _ []string) error { + if vulCmdState.Csv { + cli.EnableCSVOutput() + } + return nil + }, Short: "list the CVEs found in the hosts in your environment", Long: `List the CVEs found in the hosts in your environment. @@ -214,8 +220,14 @@ with fixes: } vulHostListHostsCmd = &cobra.Command{ - Use: "list-hosts ", - Args: cobra.ExactArgs(1), + Use: "list-hosts ", + Args: cobra.ExactArgs(1), + PreRunE: func(_ *cobra.Command, _ []string) error { + if vulCmdState.Csv { + cli.EnableCSVOutput() + } + return nil + }, Short: "list the hosts that contain a specified CVE id in your environment", Long: `List the hosts that contain a specified CVE id in your environment. @@ -242,6 +254,14 @@ To list the CVEs found in the hosts of your environment run: } rows := hostVulnHostsTable(response.Hosts) + if cli.CSVOutput() { + return cli.OutputCSV( + []string{"Machine ID", "Hostname", "External IP", "Internal IP", + "Os/Arch", "Provider", "Instance ID", "Vulnerabilities", "Status"}, + rows, + ) + } + // if the user wants to show only online or // offline hosts, show a friendly message if len(rows) == 0 { @@ -370,6 +390,11 @@ func init() { vulHostListCvesCmd.Flags(), ) + setCsvFlag( + vulHostListCvesCmd.Flags(), + vulHostListHostsCmd.Flags(), + ) + // add online flag to host list-hosts command vulHostListHostsCmd.Flags().BoolVar(&vulCmdState.Online, "online", false, "only show hosts that are online", @@ -1002,6 +1027,14 @@ func buildListCVEReports(cves []api.HostVulnCVE) error { if vulCmdState.Packages { packages, filtered := hostVulnPackagesTable(cves, true) + + if cli.CSVOutput() { + return cli.OutputCSV( + []string{"CVE Count", "Severity", "Package", "Current Version", "Fix Version", "Pkg Status", "Hosts"}, + packages, + ) + } + cli.OutputHuman( renderSimpleTable( []string{"CVE Count", "Severity", "Package", "Current Version", "Fix Version", "Pkg Status", "Hosts"}, @@ -1022,6 +1055,14 @@ func buildListCVEReports(cves []api.HostVulnCVE) error { return nil } + if cli.CSVOutput() { + return cli.OutputCSV( + []string{"CVE ID", "Severity", "Score", "Package", "Current Version", + "Fix Version", "OS Version", "Hosts", "Pkg Status", "Vuln Status"}, + rows, + ) + } + cli.OutputHuman( renderSimpleTable( []string{"CVE ID", "Severity", "Score", "Package", "Current Version", diff --git a/cli/cmd/vulnerability.go b/cli/cmd/vulnerability.go index 54a59d646..9efe718c8 100644 --- a/cli/cmd/vulnerability.go +++ b/cli/cmd/vulnerability.go @@ -40,6 +40,9 @@ var ( // store the vulnerability assessment in HTML format on disk Html bool + // output vulnerability assessment in CSV format + Csv bool + // when enabled we tread the provided sha256 hash as image id ImageID bool @@ -170,6 +173,16 @@ func setHtmlFlag(cmds ...*flag.FlagSet) { } } +func setCsvFlag(cmds ...*flag.FlagSet) { + for _, cmd := range cmds { + if cmd != nil { + cmd.BoolVar(&vulCmdState.Csv, "csv", false, + "output vulnerability assessment in CSV format", + ) + } + } +} + func setDetailsFlag(cmds ...*flag.FlagSet) { for _, cmd := range cmds { if cmd != nil {