feat(cli): Add CSV rendering output for select commands (#424)

Adds the ability to get CSV output from three commands using the `--csv` flag:

* `lacework comp [csp] get-report`
* `lacework vuln host list-cves`
* `lacework vuln host list-hosts`

JIRA: ALLY-465

cli/cmd/cli_state.go

@@ -60,6 +60,7 @@ type cliState struct {
 	workers        sync.WaitGroup
 	spinner        *spinner.Spinner
 	jsonOutput     bool
+	csvOutput      bool
 	nonInteractive bool
 	profileDetails map[string]interface{}
 }
@@ -210,7 +211,7 @@ func (c *cliState) NewClient() error {
 
 // InteractiveMode returns true if the cli is running in interactive mode
 func (c *cliState) InteractiveMode() bool {
-	return !c.nonInteractive
+	return !c.nonInteractive && !c.csvOutput
 }
 
 // NonInteractive turns off interactive mode, that is, no progress bars and spinners
@@ -274,14 +275,25 @@ func (c *cliState) EnableHumanOutput() {
 	c.jsonOutput = false
 }
 
+// EnableCSVOutput enables the cli to display CSV output
+func (c *cliState) EnableCSVOutput() {
+	c.Log.Info("switch output to csv format")
+	c.csvOutput = true
+}
+
 // JSONOutput returns true if the cli is configured to display JSON output
 func (c *cliState) JSONOutput() bool {
 	return c.jsonOutput
 }
 
-// HumanOutput returns true if the cli is configured to siplay human readable output
+// HumanOutput returns true if the cli is configured to display human readable output
 func (c *cliState) HumanOutput() bool {
-	return !c.jsonOutput
+	return !c.jsonOutput && !c.csvOutput
+}
+
+// CSVOutput returns true if the cli is configured to display csv output
+func (c *cliState) CSVOutput() bool {
+	return c.csvOutput
 }
 
 // loadStateFromViper loads parameters and environment variables

cli/cmd/compliance.go

@@ -39,6 +39,9 @@ var (
 		// download report in PDF format
 		Pdf bool
 
+		// output report in CSV format
+		Csv bool
+
 		// display extended details about a compliance report
 		Details bool
 

cli/cmd/compliance_aws.go

@@ -84,6 +84,10 @@ Then navigate to Settings > Integrations > Cloud Accounts.
 		Use:     "get-report <account_id>",
 		Aliases: []string{"get"},
 		PreRunE: func(_ *cobra.Command, _ []string) error {
+			if compCmdState.Csv {
+				cli.EnableCSVOutput()
+			}
+
 			switch compCmdState.Type {
 			case "CIS":
 				compCmdState.Type = fmt.Sprintf("AWS_%s_S3", compCmdState.Type)
@@ -176,6 +180,13 @@ To run an ad-hoc compliance assessment of an AWS account:
 			}
 
 			recommendations := complianceReportRecommendationsTable(report.Recommendations)
+			if cli.CSVOutput() {
+				return cli.OutputCSV(
+					[]string{"ID", "Recommendation", "Status", "Severity", "Service", "Affected", "Assessed"},
+					recommendations,
+				)
+			}
+
 			cli.OutputHuman("\n")
 			cli.OutputHuman(
 				buildComplianceReportTable(
@@ -236,6 +247,10 @@ func init() {
 		"download report in PDF format",
 	)
 
+	complianceAwsGetReportCmd.Flags().BoolVar(&compCmdState.Csv, "csv", false,
+		"output report in CSV format",
+	)
+
 	// AWS report types: AWS_CIS_S3, NIST_800-53_Rev4, ISO_2700, HIPAA, SOC, or PCI
 	complianceAwsGetReportCmd.Flags().StringVar(&compCmdState.Type, "type", "CIS",
 		"report type to display, supported types: CIS, NIST_800-53_Rev4, ISO_2700, HIPAA, SOC, or PCI",

cli/cmd/compliance_azure.go

@@ -128,6 +128,10 @@ Then navigate to Settings > Integrations > Cloud Accounts.
 		Use:     "get-report <tenant_id> <subscriptions_id>",
 		Aliases: []string{"get"},
 		PreRunE: func(_ *cobra.Command, _ []string) error {
+			if compCmdState.Csv {
+				cli.EnableCSVOutput()
+			}
+
 			switch compCmdState.Type {
 			case "CIS", "SOC", "PCI":
 				compCmdState.Type = fmt.Sprintf("AZURE_%s", compCmdState.Type)
@@ -218,6 +222,13 @@ To run an ad-hoc compliance assessment use the command:
 			}
 
 			recommendations := complianceReportRecommendationsTable(report.Recommendations)
+			if cli.CSVOutput() {
+				return cli.OutputCSV(
+					[]string{"ID", "Recommendation", "Status", "Severity", "Service", "Affected", "Assessed"},
+					recommendations,
+				)
+			}
+
 			cli.OutputHuman("\n")
 			cli.OutputHuman(
 				buildComplianceReportTable(
@@ -275,6 +286,11 @@ func init() {
 		"download report in PDF format",
 	)
 
+	// Output the report in CSV format
+	complianceAzureGetReportCmd.Flags().BoolVar(&compCmdState.Csv, "csv", false,
+		"output report in CSV format",
+	)
+
 	// Azure report types: AZURE_CIS, AZURE_SOC, or AZURE_PCI
 	complianceAzureGetReportCmd.Flags().StringVar(&compCmdState.Type, "type", "CIS",
 		"report type to display, supported types: CIS, SOC, or PCI",

cli/cmd/compliance_gcp.go

@@ -83,6 +83,10 @@ Then, select one GUID from an integration and visualize its details using the co
 		Use:     "get-report <organization_id> <project_id>",
 		Aliases: []string{"get"},
 		PreRunE: func(_ *cobra.Command, _ []string) error {
+			if compCmdState.Csv {
+				cli.EnableCSVOutput()
+			}
+
 			switch compCmdState.Type {
 			case "CIS", "SOC", "PCI":
 				compCmdState.Type = fmt.Sprintf("GCP_%s", compCmdState.Type)
@@ -173,6 +177,13 @@ To run an ad-hoc compliance assessment use the command:
 			}
 
 			recommendations := complianceReportRecommendationsTable(report.Recommendations)
+			if cli.CSVOutput() {
+				return cli.OutputCSV(
+					[]string{"ID", "Recommendation", "Status", "Severity", "Service", "Affected", "Assessed"},
+					recommendations,
+				)
+			}
+
 			cli.OutputHuman("\n")
 			cli.OutputHuman(
 				buildComplianceReportTable(
@@ -229,6 +240,11 @@ func init() {
 		"download report in PDF format",
 	)
 
+	// Output the report in CSV format
+	complianceGcpGetReportCmd.Flags().BoolVar(&compCmdState.Csv, "csv", false,
+		"output report in CSV format",
+	)
+
 	// GCP report types: GCP_CIS, GCP_SOC, or GCP_PCI.
 	complianceGcpGetReportCmd.Flags().StringVar(&compCmdState.Type, "type", "CIS",
 		"report type to display, supported types: CIS, SOC, or PCI",

cli/cmd/outputs.go

@@ -19,11 +19,13 @@
 package cmd
 
 import (
+	"encoding/csv"
 	"fmt"
 	"os"
 	"strings"
 
 	"github.com/fatih/color"
+	"github.com/pkg/errors"
 )
 
 // OutputJSON will print out the JSON representation of the provided data
@@ -64,3 +66,45 @@ func (c *cliState) FormatJSONString(s string) (string, error) {
 	}
 	return string(pretty), nil
 }
+
+// Used to clean CSV inputs prior to rendering
+func csvCleanData(input []string) []string {
+	var data []string
+	for _, h := range input {
+		data = append(data, strings.Replace(h, "\n", "", -1))
+	}
+	return data
+}
+
+// Used to produce CSV output
+func renderAsCSV(headers []string, data [][]string) (string, error) {
+	csvOut := &strings.Builder{}
+	csv := csv.NewWriter(csvOut)
+
+	if len(headers) > 0 {
+		if err := csv.Write(csvCleanData(headers)); err != nil {
+			return "", errors.Wrap(err, "Failed to build csv output")
+		}
+	}
+
+	for _, record := range data {
+		if err := csv.Write(csvCleanData(record)); err != nil {
+			return "", errors.Wrap(err, "Failed to build csv output")
+		}
+	}
+
+	// Write any buffered data to the underlying writer (standard output).
+	csv.Flush()
+	return csvOut.String(), csv.Error()
+}
+
+// OutputCSV will print out the provided headers/data in CSV format
+func (c *cliState) OutputCSV(headers []string, data [][]string) error {
+	csv, err := renderAsCSV(headers, data)
+	if err != nil {
+		return err
+	}
+
+	fmt.Fprint(os.Stdout, csv)
+	return nil
+}

cli/cmd/outputs_test.go

@@ -0,0 +1,75 @@
+//
+// Author:: Matt Cadorette (<[email protected]>)
+// Copyright:: Copyright 2020, Lacework Inc.
+// License:: Apache License, Version 2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package cmd
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestRenderSimpleCSV(t *testing.T) {
+	expectedCsv := strings.TrimPrefix(`
+KEY,VALUE
+key1,value1
+key2,value2
+key3,value3
+`, "\n")
+
+	csv, _ := renderAsCSV(
+		[]string{"KEY", "VALUE"},
+		[][]string{
+			{"key1", "value1"},
+			{"key2", "value2"},
+			{"key3", "value3"},
+		},
+	)
+	assert.Equal(t,
+		csv,
+		expectedCsv,
+		"csv is not being formatted correctly")
+}
+
+func TestRenderComplexCSV(t *testing.T) {
+	expectedCsv := strings.TrimPrefix(`
+KEY HEADER VALUE,"VALUE,TEST"
+key1,"this is a value, from [a, b, c]"
+key2,value2
+key3,value3
+`, "\n")
+
+	csv, _ := renderAsCSV(
+		[]string{"KEY\n HEADER VALUE", "VALUE,TEST"},
+		[][]string{
+			{"key1", "this is a value, from [a, b, c]"},
+			{"key2", "value2"},
+			{"key3", "value3"},
+		},
+	)
+	assert.Equal(t,
+		csv,
+		expectedCsv,
+		"csv is not being formatted correctly")
+}
+
+func TestCSVDataCleanup(t *testing.T) {
+	data := csvCleanData([]string{"KEY\n HEADER\n VALUE", "VALUE,TEST\n"})
+	assert.NotContains(t, strings.Join(data, ""), "\n", "data is not being cleaned up properly")
+}

cli/cmd/vuln_host.go

@@ -187,8 +187,14 @@ To generate a package-manifest from the local host and scan it automatically:
 	}
 
 	vulHostListCvesCmd = &cobra.Command{
-		Use:   "list-cves",
-		Args:  cobra.NoArgs,
+		Use:  "list-cves",
+		Args: cobra.NoArgs,
+		PreRunE: func(_ *cobra.Command, _ []string) error {
+			if vulCmdState.Csv {
+				cli.EnableCSVOutput()
+			}
+			return nil
+		},
 		Short: "list the CVEs found in the hosts in your environment",
 		Long: `List the CVEs found in the hosts in your environment.
 
@@ -214,8 +220,14 @@ with fixes:
 	}
 
 	vulHostListHostsCmd = &cobra.Command{
-		Use:   "list-hosts <cve_id>",
-		Args:  cobra.ExactArgs(1),
+		Use:  "list-hosts <cve_id>",
+		Args: cobra.ExactArgs(1),
+		PreRunE: func(_ *cobra.Command, _ []string) error {
+			if vulCmdState.Csv {
+				cli.EnableCSVOutput()
+			}
+			return nil
+		},
 		Short: "list the hosts that contain a specified CVE id in your environment",
 		Long: `List the hosts that contain a specified CVE id in your environment.
 
@@ -242,6 +254,14 @@ To list the CVEs found in the hosts of your environment run:
 			}
 
 			rows := hostVulnHostsTable(response.Hosts)
+			if cli.CSVOutput() {
+				return cli.OutputCSV(
+					[]string{"Machine ID", "Hostname", "External IP", "Internal IP",
+						"Os/Arch", "Provider", "Instance ID", "Vulnerabilities", "Status"},
+					rows,
+				)
+			}
+
 			// if the user wants to show only online or
 			// offline hosts, show a friendly message
 			if len(rows) == 0 {
@@ -370,6 +390,11 @@ func init() {
 		vulHostListCvesCmd.Flags(),
 	)
 
+	setCsvFlag(
+		vulHostListCvesCmd.Flags(),
+		vulHostListHostsCmd.Flags(),
+	)
+
 	// add online flag to host list-hosts command
 	vulHostListHostsCmd.Flags().BoolVar(&vulCmdState.Online,
 		"online", false, "only show hosts that are online",
@@ -1002,6 +1027,14 @@ func buildListCVEReports(cves []api.HostVulnCVE) error {
 
 	if vulCmdState.Packages {
 		packages, filtered := hostVulnPackagesTable(cves, true)
+
+		if cli.CSVOutput() {
+			return cli.OutputCSV(
+				[]string{"CVE Count", "Severity", "Package", "Current Version", "Fix Version", "Pkg Status", "Hosts"},
+				packages,
+			)
+		}
+
 		cli.OutputHuman(
 			renderSimpleTable(
 				[]string{"CVE Count", "Severity", "Package", "Current Version", "Fix Version", "Pkg Status", "Hosts"},
@@ -1022,6 +1055,14 @@ func buildListCVEReports(cves []api.HostVulnCVE) error {
 		return nil
 	}
 
+	if cli.CSVOutput() {
+		return cli.OutputCSV(
+			[]string{"CVE ID", "Severity", "Score", "Package", "Current Version",
+				"Fix Version", "OS Version", "Hosts", "Pkg Status", "Vuln Status"},
+			rows,
+		)
+	}
+
 	cli.OutputHuman(
 		renderSimpleTable(
 			[]string{"CVE ID", "Severity", "Score", "Package", "Current Version",