Setup initial E2E tests closes #2848

.github/workflows/go-ci.yml

@@ -58,8 +58,6 @@ jobs:
       uses: actions/checkout@v2
       with:
         persist-credentials: false
-    - name: Run Go mod tidy
-      run: go mod tidy
     - name: Get cache paths
       id: go-cache-paths
       run: |
@@ -85,7 +83,7 @@ jobs:
     - name: Test and Generate Report
       run: |
         set +o pipefail
-        go test -mod=vendor -v ./... -count=1 -coverprofile cover.out 2>&1 | go-junit-report -set-exit-code -go-version ${{ matrix.go-version }} -package-name "github.com/Checkmarx/kics/test" > test-report-${{ matrix.os }}.xml
+        go test -mod=vendor -v $(go list ./... | grep -v e2e) -count=1 -coverprofile cover.out 2>&1 | go-junit-report -set-exit-code -go-version ${{ matrix.go-version }} -package-name "github.com/Checkmarx/kics/test" > test-report-${{ matrix.os }}.xml
     - name: Archive unit tests report
       uses: actions/upload-artifact@v2
       with:

.github/workflows/go-e2e.yaml

@@ -0,0 +1,61 @@
+name: go-e2e
+
+on:
+  pull_request:
+    branches: [master]
+
+jobs:
+  unit-tests:
+    name: e2e-tests
+    strategy:
+      matrix:
+        go-version: [1.16.x]
+        os: [ubuntu-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Cancel Previous Runs
+        uses: styfle/[email protected]
+        with:
+          access_token: ${{ github.token }}
+      - name: Set up Go 1.x
+        uses: actions/setup-go@v2
+        with:
+          go-version: ${{ matrix.go-version }}
+      - name: Check out code
+        uses: actions/checkout@v2
+        with:
+          persist-credentials: false
+      - name: Print go env
+        run: go env
+      - name: Get cache paths
+        id: go-cache-paths
+        run: |
+          echo "::set-output name=go-build::$(go env GOCACHE)"
+          echo "::set-output name=go-mod::$(go env GOMODCACHE)"
+      - name: Cache dependencies
+        uses: actions/[email protected]
+        with:
+          path: ${{ steps.go-cache-paths.outputs.go-build }}
+          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.OS }}-build-${{ env.cache-name }}
+            ${{ runner.OS }}-build-
+            ${{ runner.OS }}-
+      - name: Build binary
+        run: make build
+      - name: Get Binary Path
+        id: getbin
+        run: |
+          #!/usr/bin/env python3
+          import os
+          path = os.path.join(os.environ['GITHUB_WORKSPACE'], 'bin', 'kics')
+          print(f"::set-output name=kics::{path}")
+          queries_path = os.path.join(os.environ['GITHUB_WORKSPACE'], 'assets', 'queries')
+          print(f"::set-output name=queries::{queries_path}")
+        shell: python3 {0}
+      - name: Run E2E Tests
+        env:
+          E2E_KICS_BINARY: ${{ steps.getbin.outputs.kics }}
+          E2E_KICS_QUERIES_PATH: ${{ steps.getbin.outputs.queries }}
+        run: |
+          go test "github.com/Checkmarx/kics/e2e" -v

.golangci.yml

@@ -111,6 +111,7 @@ run:
     - docs
     - vendor
     - tools
+    - e2e
 
 # golangci.com configuration
 # https://github.com/golangci/golangci/wiki/Configuration

Makefile

@@ -49,7 +49,7 @@ build-all: lint generate
 
 .PHONY: build
 build: ## go build
-build: lint generate
+build: generate
 	$(call print-target)
 	@go build -o ${TARGET_BIN} \
 		-ldflags "-X github.com/Checkmarx/kics/internal/constants.Version=${VERSION} -X github.com/Checkmarx/kics/internal/constants.SCMCommit=${COMMIT}" \

e2e/cli_test.go

@@ -0,0 +1,217 @@
+package e2e
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"reflect"
+	"strings"
+	"testing"
+
+	"github.com/Checkmarx/kics/pkg/model"
+	"github.com/Checkmarx/kics/test"
+	"github.com/stretchr/testify/require"
+)
+
+type logMsg struct {
+	Level    string `json:"level"`
+	ErrorMgs string `json:"error"`
+	Message  string `json:"message"`
+}
+
+type cmdArgs []string
+
+type args struct {
+	args            []cmdArgs // args to pass to kics binary
+	expectedOut     []string  // path to file with expected output
+	expectedPayload []string
+}
+
+var tests = []struct {
+	name          string
+	args          args
+	wantStatus    int
+	removePayload []string
+}{
+	// E2E_CLI_001 - KICS command should display a help text in the CLI when provided with the
+	// 	 --help flag and it should describe the available commands plus the global flags
+	{
+		name: "E2E_CLI_001",
+		args: args{
+			args: []cmdArgs{
+				[]string{"--help"},
+			},
+			expectedOut:     []string{"E2E_CLI_001"},
+			expectedPayload: []string{},
+		},
+		removePayload: []string{},
+		wantStatus:    0,
+	},
+	// E2E-CLI-002 - KICS scan command should display a help text in the CLI when provided with the
+	// --help flag and it should describe the options related with scan plus the global options
+	{
+		name: "E2E-CLI-002",
+		args: args{
+			args: []cmdArgs{
+				[]string{"scan", "--help"},
+			},
+			expectedOut: []string{"E2E_CLI_002"},
+		},
+		wantStatus: 0,
+	},
+	// E2E-CLI-003 - KICS scan command had a mandatory flag -p the CLI should exhibit
+	// an error message and return exit code 1
+	{
+		name: "E2E-CLI-003",
+		args: args{
+			args: []cmdArgs{
+				[]string{"scan"},
+			},
+			expectedOut: []string{"E2E_CLI_003"},
+		},
+		wantStatus: 1,
+	},
+	// E2E-CLI-004 - KICS scan command had a mandatory flag -p the CLI should exhibit
+	// an error message and return exit code 1
+	{
+		name: "E2E-CLI-004",
+		args: args{
+			args: []cmdArgs{
+				[]string{"--ci", "--verbose"},
+				[]string{"scan", "--ci", "--verbose"},
+				[]string{"--ci", "scan", "--verbose"},
+			},
+			expectedOut: []string{
+				"E2E_CLI_004",
+				"E2E_CLI_004",
+				"E2E_CLI_004",
+			},
+		},
+		wantStatus: 1,
+	},
+	// E2E-CLI-005 - KICS scan with -- payload-path flag should create a file with the
+	// passed name containing the payload of the files scanned
+	{
+		name: "E2E-CLI-005",
+		args: args{
+			args: []cmdArgs{
+				[]string{"scan", "--silent", "-q", "../assets/queries", "-p", "fixtures/samples/terraform.tf",
+					"--payload-path", "fixtures/payload.json", "-q", "../assets/queries"},
+			},
+			expectedOut: []string{
+				"E2E_CLI_005",
+			},
+			expectedPayload: []string{
+				"E2E_CLI_005_PAYLOAD",
+			},
+		},
+		wantStatus:    0,
+		removePayload: []string{"payload.json"},
+	},
+}
+
+func Test_E2E_CLI(t *testing.T) {
+	kicsPath := getKICSBinaryPath("")
+
+	for _, tt := range tests {
+		for arg := range tt.args.args {
+			t.Run(fmt.Sprintf("%s_%d", tt.name, arg), func(t *testing.T) {
+				out, err := runCommand(append(kicsPath, tt.args.args[arg]...))
+				// Check command Error
+				require.NoError(t, err, "Capture output should not yield an error")
+				// Check exit status code
+				if !reflect.DeepEqual(out.status, tt.wantStatus) {
+					t.Errorf("kics status = %v, want status = %v", out.status, tt.wantStatus)
+				}
+				// Get and preapare expected output
+				want, err := prepareExpected(tt.args.expectedOut[arg])
+				require.NoError(t, err, "Reading a fixture should not yield an error")
+				// Check Number of Lines
+				require.Equal(t, len(want), len(out.output),
+					"\nExpected number of stdout lines:%d\nActual of stdout lines:%d\n", len(want), len(out.output))
+				// Check output lines
+				for idx := range want {
+					checkLine(t, out.output[idx], want[idx], idx+1)
+				}
+				// Check payload files
+				for _, file := range tt.removePayload {
+					fileCheck(t, file, tt.args.expectedPayload[arg])
+				}
+			})
+		}
+	}
+}
+
+func prepareExpected(path string) ([]string, error) {
+	cont, err := readFixture(path)
+	if err != nil {
+		return []string{}, err
+	}
+	return strings.Split(cont, "\n"), nil
+}
+
+func checkLine(t *testing.T, expec, want string, line int) {
+	logExp := logMsg{}
+	logWant := logMsg{}
+	errE := json.Unmarshal([]byte(expec), &logExp)
+	errW := json.Unmarshal([]byte(want), &logWant)
+	if errE == nil && errW == nil {
+		checkJSONLog(t, logExp, logWant)
+	} else {
+		require.Equal(t, expec, want,
+			"\nExpected Output line\n%s\nKICS Output line:\n%s\n line: %d", want, expec, line)
+	}
+}
+
+func checkJSONLog(t *testing.T, expec, want logMsg) {
+	require.Equal(t, expec.Level, want.Level,
+		"\nExpected Output line log level\n%s\nKICS Output line log level:\n%s\n", want.Level, expec.Level)
+	require.Equal(t, expec.ErrorMgs, want.ErrorMgs,
+		"\nExpected Output line error msg\n%s\nKICS Output line error msg:\n%s\n", expec.ErrorMgs, want.ErrorMgs)
+	require.Equal(t, expec.Message, want.Message,
+		"\nExpected Output line msg\n%s\nKICS Output line msg:\n%s\n", expec.Message, want.Message)
+}
+
+func fileCheck(t *testing.T, remove, payload string) {
+	wantPayload, err := prepareExpected(payload)
+	require.NoError(t, err, "Reading a fixture should not yield an error")
+	expectPayload, err := prepareExpected(remove)
+	require.NoError(t, err, "Reading a fixture should not yield an error")
+	require.Equal(t, len(wantPayload), len(expectPayload),
+		"\nExpected file number of lines:%d\nKics file number of lines:%d\n", len(wantPayload), len(expectPayload))
+	checkJSONFile(t, wantPayload, expectPayload)
+	err = os.Remove(filepath.Join("fixtures", remove))
+	require.NoError(t, err)
+}
+
+func checkJSONFile(t *testing.T, expect, want []string) { // Needs to fixed
+	var wantI model.Documents
+	var expecI model.Documents
+	errE := json.Unmarshal([]byte(strings.Join(expect, "\n")), &expecI)
+	require.NoError(t, errE, "Unmarshaling JSON file should not yield an error")
+	errW := json.Unmarshal([]byte(strings.Join(want, "\n")), &wantI)
+	require.NoError(t, errW, "Unmarshaling JSON file should not yield an error")
+	setFields(t, wantI, expecI, "payload")
+}
+
+func setFields(t *testing.T, want, expect model.Documents, location string) {
+	switch location {
+	case "payload":
+		for _, docs := range want.Documents {
+			require.NotNil(t, docs["id"]) // Here additional checks may be added as length of id, or contains in file
+			require.NotNil(t, docs["file"])
+			docs["id"] = "0"
+			docs["file"] = "file"
+		}
+		if !reflect.DeepEqual(expect, want) {
+			expectStr, err := test.StringifyStruct(expect)
+			require.NoError(t, err)
+			wantStr, err := test.StringifyStruct(want)
+			require.NoError(t, err)
+			t.Errorf("Expected:\n%v\n,want:\n%v\n", expectStr, wantStr)
+		}
+	case "result": // TODO
+	default:
+	}
+}

e2e/fixtures/E2E_CLI_001

@@ -0,0 +1,23 @@
+Keeping Infrastructure as Code Secure
+
+Usage:
+  kics [command]
+
+Available Commands:
+  generate-id    Generates uuid for query
+  help           Help about any command
+  list-platforms List supported platforms
+  scan           Executes a scan analysis
+  version        Displays the current version
+
+Flags:
+      --ci                  display only log messages to CLI output (mutually exclusive with silent)
+  -h, --help                help for kics
+  -f, --log-format string   determines log format (pretty,json) (default "pretty")
+      --log-level string    determines log level (TRACE,DEBUG,INFO,WARN,ERROR,FATAL) (default "INFO")
+      --log-path string     path to log files, (defaults to ${PWD}/info.log)
+      --no-color            disable CLI color output
+  -s, --silent              silence stdout messages (mutually exclusive with verbose and ci)
+  -v, --verbose             write logs to stdout too (mutually exclusive with silent)
+
+Use "kics [command] --help" for more information about a command.

e2e/fixtures/E2E_CLI_002

@@ -0,0 +1,39 @@
+Executes a scan analysis
+
+Usage:
+  kics scan [flags]
+
+Flags:
+      --config string                path to configuration file
+      --exclude-categories strings   exclude categories by providing its name
+                                     can be provided multiple times or as a comma separated string
+                                     example: 'Access control,Best practices'
+  -e, --exclude-paths strings        exclude paths from scan
+                                     supports glob and can be provided multiple times or as a quoted comma separated string
+                                     example: './shouldNotScan/*,somefile.txt'
+      --exclude-queries strings      exclude queries by providing the query ID
+                                     can be provided multiple times or as a comma separated string
+                                     example: 'e69890e6-fce5-461d-98ad-cb98318dfc96,4728cd65-a20c-49da-8b31-9c08b423e4db'
+  -x, --exclude-results strings      exclude results by providing the similarity ID of a result
+                                     can be provided multiple times or as a comma separated string
+                                     example: 'fec62a97d569662093dbb9739360942f...,31263s5696620s93dbb973d9360942fc2a...'
+  -h, --help                         help for scan
+      --minimal-ui                   simplified version of CLI output
+      --no-progress                  hides the progress bar
+  -o, --output-path string           directory path to store reports
+  -p, --path string                  path or directory path to scan
+  -d, --payload-path string          path to store internal representation JSON file
+      --preview-lines int            number of lines to be display in CLI results (min: 1, max: 30) (default 3)
+  -q, --queries-path string          path to directory with queries (default "./assets/queries")
+      --report-formats strings       formats in which the results will be exported (json, sarif, html)
+  -t, --type strings                 case insensitive list of platform types to scan
+                                     (Ansible, CloudFormation, Dockerfile, Kubernetes, OpenAPI, Terraform)
+
+Global Flags:
+      --ci                  display only log messages to CLI output (mutually exclusive with silent)
+  -f, --log-format string   determines log format (pretty,json) (default "pretty")
+      --log-level string    determines log level (TRACE,DEBUG,INFO,WARN,ERROR,FATAL) (default "INFO")
+      --log-path string     path to log files, (defaults to ${PWD}/info.log)
+      --no-color            disable CLI color output
+  -s, --silent              silence stdout messages (mutually exclusive with verbose and ci)
+  -v, --verbose             write logs to stdout too (mutually exclusive with silent)