From 078a470d0b93845970a9b2ff9e4cdcbf4b577760 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rog=C3=A9rio=20Peixoto?= Date: Thu, 15 Apr 2021 15:47:29 +0100 Subject: [PATCH 01/13] setup initial e2e tests closes #2848 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Rogério Peixoto --- .github/workflows/go-ci.yml | 2 +- .github/workflows/go-e2e.yaml | 35 ++++++++++++++++++++++++++++ .golangci.yml | 1 + Makefile | 2 +- e2e/cli_test.go | 24 +++++++++++++++++++ e2e/fixtures/E2E_CLI_001 | 23 +++++++++++++++++++ e2e/utils.go | 43 +++++++++++++++++++++++++++++++++++ 7 files changed, 128 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/go-e2e.yaml create mode 100644 e2e/cli_test.go create mode 100644 e2e/fixtures/E2E_CLI_001 create mode 100644 e2e/utils.go diff --git a/.github/workflows/go-ci.yml b/.github/workflows/go-ci.yml index a4545e409c4..1f880aa816a 100644 --- a/.github/workflows/go-ci.yml +++ b/.github/workflows/go-ci.yml @@ -83,7 +83,7 @@ jobs: go mod vendor - name: Test and Generate Report run: | - go test -mod=vendor -v ./... -count=1 -coverprofile cover.out 2>&1 | go-junit-report -set-exit-code -go-version ${{ matrix.go-version }} -package-name "github.com/Checkmarx/kics/test" > test-report-${{ matrix.os }}.xml + go test -mod=vendor -v $(go list ./... | grep -v e2e/) -count=1 -coverprofile cover.out 2>&1 | go-junit-report -set-exit-code -go-version ${{ matrix.go-version }} -package-name "github.com/Checkmarx/kics/test" > test-report-${{ matrix.os }}.xml - name: Archive unit tests report uses: actions/upload-artifact@v2 with: diff --git a/.github/workflows/go-e2e.yaml b/.github/workflows/go-e2e.yaml new file mode 100644 index 00000000000..a6199e3af17 --- /dev/null +++ b/.github/workflows/go-e2e.yaml @@ -0,0 +1,35 @@ + +name: go-e2e + +on: + pull_request: + branches: [master] + +jobs: + unit-tests: + name: e2e-tests + strategy: + matrix: + go-version: [1.16.x] + os: [ubuntu-latest, windows-latest, macos-latest] + runs-on: ${{ matrix.os }} + steps: + - name: Cancel Previous Runs + uses: styfle/cancel-workflow-action@0.9.0 + with: + access_token: ${{ github.token }} + - name: Set up Go 1.x + uses: actions/setup-go@v2 + with: + go-version: ${{ matrix.go-version }} + - name: Check out code + uses: actions/checkout@v2 + with: + persist-credentials: false + - name: Build binary + run: make build + - name: Run E2E Tests + env: + E2E_KICS_BINARY: ./bin/kics + run: | + go test "github.com/Checkmarx/kics/e2e" -v diff --git a/.golangci.yml b/.golangci.yml index 660f4d25c97..53a0a984665 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -111,6 +111,7 @@ run: - docs - vendor - tools + - e2e # golangci.com configuration # https://github.com/golangci/golangci/wiki/Configuration diff --git a/Makefile b/Makefile index e2f640806c4..ee5c34a9fd2 100644 --- a/Makefile +++ b/Makefile @@ -49,7 +49,7 @@ build-all: lint generate .PHONY: build build: ## go build -build: lint generate +build: generate $(call print-target) @go build -o ${TARGET_BIN} \ -ldflags "-X github.com/Checkmarx/kics/internal/constants.Version=${VERSION} -X github.com/Checkmarx/kics/internal/constants.SCMCommit=${COMMIT}" \ diff --git a/e2e/cli_test.go b/e2e/cli_test.go new file mode 100644 index 00000000000..21fc4f9e15a --- /dev/null +++ b/e2e/cli_test.go @@ -0,0 +1,24 @@ +package e2e + +import ( + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +// E2E_CLI_001 - KICS command should display a help text in the CLI when provided with the --help flag and it should describe the available commands plus the global flags +func Test_E2E_CLI_001(t *testing.T) { + kicsPath := getKICSBinaryPath("") + actualOutput, err := runCommandAndReturnOutput([]string{kicsPath, "--help"}) + require.NoError(t, err, "Capture output should not yield an error") + actualLines := strings.Split(actualOutput, "\n") + + expectedOutput, err := readFixture("E2E_CLI_001") + require.NoError(t, err, "Reading a fixture should not yield an error") + expectedLines := strings.Split(expectedOutput, "\n") + for idx := range expectedLines { + require.Equal(t, expectedLines[idx], actualLines[idx], fmt.Sprintf("Expected output line\n%s is not equal to actual output line\n%s\n line: %d", expectedLines[idx], actualLines[idx], idx)) + } +} diff --git a/e2e/fixtures/E2E_CLI_001 b/e2e/fixtures/E2E_CLI_001 new file mode 100644 index 00000000000..36982dc2acc --- /dev/null +++ b/e2e/fixtures/E2E_CLI_001 @@ -0,0 +1,23 @@ +Keeping Infrastructure as Code Secure + +Usage: + kics [command] + +Available Commands: + generate-id Generates uuid for query + help Help about any command + list-platforms List supported platforms + scan Executes a scan analysis + version Displays the current version + +Flags: + --ci display only log messages to CLI output (mutually exclusive with silent) + -h, --help help for kics + -f, --log-format string determines log format (pretty,json) (default "pretty") + --log-level string determines log level (TRACE,DEBUG,INFO,WARN,ERROR,FATAL) (default "INFO") + --log-path string path to log files, (defaults to ${PWD}/info.log) + --no-color disable CLI color output + -s, --silent silence stdout messages (mutually exclusive with verbose and ci) + -v, --verbose write logs to stdout too (mutually exclusive with silent) + +Use "kics [command] --help" for more information about a command. \ No newline at end of file diff --git a/e2e/utils.go b/e2e/utils.go new file mode 100644 index 00000000000..447af055508 --- /dev/null +++ b/e2e/utils.go @@ -0,0 +1,43 @@ +package e2e + +import ( + "io" + "os" + "os/exec" + "path/filepath" +) + +func runCommandAndReturnOutput(args []string) (stdout string, err error) { + cmd := exec.Command(args[0], args[1:]...) //nolint + stdOutput, err := cmd.Output() + if err != nil { + return "", err + } + return string(stdOutput), nil +} + +func readFixture(testName string) (string, error) { + return readFile(filepath.Join("fixtures", testName)) +} + +func readFile(path string) (string, error) { + ostat, err := os.Open(filepath.Clean(path)) + if err != nil { + return "", err + } + bytes, err := io.ReadAll(ostat) + if err != nil { + return "", err + } + return string(bytes), nil +} + +func getKICSBinaryPath(path string) string { + var rtnPath string + if path == "" { + rtnPath = os.Getenv("E2E_KICS_BINARY") + } else { + rtnPath = path + } + return rtnPath +} From c02c716efff25bd0d8da7c3703c47aa8d0e0236b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rog=C3=A9rio=20Peixoto?= Date: Thu, 15 Apr 2021 15:47:29 +0100 Subject: [PATCH 02/13] setup initial e2e tests closes #2848 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Rogério Peixoto --- .github/workflows/go-ci.yml | 2 +- .github/workflows/go-e2e.yaml | 38 +++++++++++++++++++++++++++++++ .golangci.yml | 1 + Makefile | 2 +- e2e/cli_test.go | 24 +++++++++++++++++++ e2e/fixtures/E2E_CLI_001 | 23 +++++++++++++++++++ e2e/utils.go | 43 +++++++++++++++++++++++++++++++++++ 7 files changed, 131 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/go-e2e.yaml create mode 100644 e2e/cli_test.go create mode 100644 e2e/fixtures/E2E_CLI_001 create mode 100644 e2e/utils.go diff --git a/.github/workflows/go-ci.yml b/.github/workflows/go-ci.yml index a4545e409c4..1f880aa816a 100644 --- a/.github/workflows/go-ci.yml +++ b/.github/workflows/go-ci.yml @@ -83,7 +83,7 @@ jobs: go mod vendor - name: Test and Generate Report run: | - go test -mod=vendor -v ./... -count=1 -coverprofile cover.out 2>&1 | go-junit-report -set-exit-code -go-version ${{ matrix.go-version }} -package-name "github.com/Checkmarx/kics/test" > test-report-${{ matrix.os }}.xml + go test -mod=vendor -v $(go list ./... | grep -v e2e/) -count=1 -coverprofile cover.out 2>&1 | go-junit-report -set-exit-code -go-version ${{ matrix.go-version }} -package-name "github.com/Checkmarx/kics/test" > test-report-${{ matrix.os }}.xml - name: Archive unit tests report uses: actions/upload-artifact@v2 with: diff --git a/.github/workflows/go-e2e.yaml b/.github/workflows/go-e2e.yaml new file mode 100644 index 00000000000..16aae75e4c4 --- /dev/null +++ b/.github/workflows/go-e2e.yaml @@ -0,0 +1,38 @@ + +name: go-e2e + +on: + pull_request: + branches: [master] + +jobs: + unit-tests: + name: e2e-tests + strategy: + matrix: + go-version: [1.16.x] + os: [ubuntu-latest, windows-latest, macos-latest] + runs-on: ${{ matrix.os }} + steps: + - name: Cancel Previous Runs + uses: styfle/cancel-workflow-action@0.9.0 + with: + access_token: ${{ github.token }} + - name: Set up Go 1.x + uses: actions/setup-go@v2 + with: + go-version: ${{ matrix.go-version }} + - name: Check out code + uses: actions/checkout@v2 + with: + persist-credentials: false + - name: Build binary + run: make build + - name: Run E2E Tests + env: + E2E_KICS_BINARY: ${PWD}/bin/kics + run: | + env + ls -la + ls -la bin + go test "github.com/Checkmarx/kics/e2e" -v diff --git a/.golangci.yml b/.golangci.yml index 660f4d25c97..53a0a984665 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -111,6 +111,7 @@ run: - docs - vendor - tools + - e2e # golangci.com configuration # https://github.com/golangci/golangci/wiki/Configuration diff --git a/Makefile b/Makefile index e2f640806c4..ee5c34a9fd2 100644 --- a/Makefile +++ b/Makefile @@ -49,7 +49,7 @@ build-all: lint generate .PHONY: build build: ## go build -build: lint generate +build: generate $(call print-target) @go build -o ${TARGET_BIN} \ -ldflags "-X github.com/Checkmarx/kics/internal/constants.Version=${VERSION} -X github.com/Checkmarx/kics/internal/constants.SCMCommit=${COMMIT}" \ diff --git a/e2e/cli_test.go b/e2e/cli_test.go new file mode 100644 index 00000000000..21fc4f9e15a --- /dev/null +++ b/e2e/cli_test.go @@ -0,0 +1,24 @@ +package e2e + +import ( + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +// E2E_CLI_001 - KICS command should display a help text in the CLI when provided with the --help flag and it should describe the available commands plus the global flags +func Test_E2E_CLI_001(t *testing.T) { + kicsPath := getKICSBinaryPath("") + actualOutput, err := runCommandAndReturnOutput([]string{kicsPath, "--help"}) + require.NoError(t, err, "Capture output should not yield an error") + actualLines := strings.Split(actualOutput, "\n") + + expectedOutput, err := readFixture("E2E_CLI_001") + require.NoError(t, err, "Reading a fixture should not yield an error") + expectedLines := strings.Split(expectedOutput, "\n") + for idx := range expectedLines { + require.Equal(t, expectedLines[idx], actualLines[idx], fmt.Sprintf("Expected output line\n%s is not equal to actual output line\n%s\n line: %d", expectedLines[idx], actualLines[idx], idx)) + } +} diff --git a/e2e/fixtures/E2E_CLI_001 b/e2e/fixtures/E2E_CLI_001 new file mode 100644 index 00000000000..36982dc2acc --- /dev/null +++ b/e2e/fixtures/E2E_CLI_001 @@ -0,0 +1,23 @@ +Keeping Infrastructure as Code Secure + +Usage: + kics [command] + +Available Commands: + generate-id Generates uuid for query + help Help about any command + list-platforms List supported platforms + scan Executes a scan analysis + version Displays the current version + +Flags: + --ci display only log messages to CLI output (mutually exclusive with silent) + -h, --help help for kics + -f, --log-format string determines log format (pretty,json) (default "pretty") + --log-level string determines log level (TRACE,DEBUG,INFO,WARN,ERROR,FATAL) (default "INFO") + --log-path string path to log files, (defaults to ${PWD}/info.log) + --no-color disable CLI color output + -s, --silent silence stdout messages (mutually exclusive with verbose and ci) + -v, --verbose write logs to stdout too (mutually exclusive with silent) + +Use "kics [command] --help" for more information about a command. \ No newline at end of file diff --git a/e2e/utils.go b/e2e/utils.go new file mode 100644 index 00000000000..447af055508 --- /dev/null +++ b/e2e/utils.go @@ -0,0 +1,43 @@ +package e2e + +import ( + "io" + "os" + "os/exec" + "path/filepath" +) + +func runCommandAndReturnOutput(args []string) (stdout string, err error) { + cmd := exec.Command(args[0], args[1:]...) //nolint + stdOutput, err := cmd.Output() + if err != nil { + return "", err + } + return string(stdOutput), nil +} + +func readFixture(testName string) (string, error) { + return readFile(filepath.Join("fixtures", testName)) +} + +func readFile(path string) (string, error) { + ostat, err := os.Open(filepath.Clean(path)) + if err != nil { + return "", err + } + bytes, err := io.ReadAll(ostat) + if err != nil { + return "", err + } + return string(bytes), nil +} + +func getKICSBinaryPath(path string) string { + var rtnPath string + if path == "" { + rtnPath = os.Getenv("E2E_KICS_BINARY") + } else { + rtnPath = path + } + return rtnPath +} From f0e931bef25134f2186fe24ba47186cc103cf0f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rog=C3=A9rio=20Peixoto?= Date: Fri, 16 Apr 2021 14:13:10 +0100 Subject: [PATCH 03/13] fixing pipelines --- .github/workflows/go-ci.yml | 2 +- .github/workflows/go-e2e.yaml | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/go-ci.yml b/.github/workflows/go-ci.yml index 12ad58d1096..7fea0711808 100644 --- a/.github/workflows/go-ci.yml +++ b/.github/workflows/go-ci.yml @@ -84,7 +84,7 @@ jobs: go mod vendor - name: Test and Generate Report run: | - go test -mod=vendor -v $(go list ./... | grep -v e2e/) -count=1 -coverprofile cover.out 2>&1 | go-junit-report -set-exit-code -go-version ${{ matrix.go-version }} -package-name "github.com/Checkmarx/kics/test" > test-report-${{ matrix.os }}.xml + go test -mod=vendor -v $(go list ./... | grep -v e2e) -count=1 -coverprofile cover.out 2>&1 | go-junit-report -set-exit-code -go-version ${{ matrix.go-version }} -package-name "github.com/Checkmarx/kics/test" > test-report-${{ matrix.os }}.xml - name: Archive unit tests report uses: actions/upload-artifact@v2 with: diff --git a/.github/workflows/go-e2e.yaml b/.github/workflows/go-e2e.yaml index 16aae75e4c4..ac55e170c6f 100644 --- a/.github/workflows/go-e2e.yaml +++ b/.github/workflows/go-e2e.yaml @@ -30,9 +30,7 @@ jobs: run: make build - name: Run E2E Tests env: - E2E_KICS_BINARY: ${PWD}/bin/kics + E2E_KICS_BINARY: ${{ github.workspace }}/bin/kics run: | env - ls -la - ls -la bin go test "github.com/Checkmarx/kics/e2e" -v From d76e1a5cb5720bff86df8bc596626b800bf0b230 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rog=C3=A9rio=20Peixoto?= Date: Fri, 16 Apr 2021 14:20:46 +0100 Subject: [PATCH 04/13] adding cache to pipeline --- .github/workflows/go-ci.yml | 3 +-- .github/workflows/go-e2e.yaml | 17 ++++++++++++++++- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/.github/workflows/go-ci.yml b/.github/workflows/go-ci.yml index 7fea0711808..c6fa01a1c95 100644 --- a/.github/workflows/go-ci.yml +++ b/.github/workflows/go-ci.yml @@ -58,8 +58,6 @@ jobs: uses: actions/checkout@v2 with: persist-credentials: false - - name: Run Go mod tidy - run: go mod tidy - name: Get cache paths id: go-cache-paths run: | @@ -84,6 +82,7 @@ jobs: go mod vendor - name: Test and Generate Report run: | + set +o pipefail go test -mod=vendor -v $(go list ./... | grep -v e2e) -count=1 -coverprofile cover.out 2>&1 | go-junit-report -set-exit-code -go-version ${{ matrix.go-version }} -package-name "github.com/Checkmarx/kics/test" > test-report-${{ matrix.os }}.xml - name: Archive unit tests report uses: actions/upload-artifact@v2 diff --git a/.github/workflows/go-e2e.yaml b/.github/workflows/go-e2e.yaml index ac55e170c6f..1b2d759afd1 100644 --- a/.github/workflows/go-e2e.yaml +++ b/.github/workflows/go-e2e.yaml @@ -26,11 +26,26 @@ jobs: uses: actions/checkout@v2 with: persist-credentials: false + - name: Print go env + run: go env + - name: Get cache paths + id: go-cache-paths + run: | + echo "::set-output name=go-build::$(go env GOCACHE)" + echo "::set-output name=go-mod::$(go env GOMODCACHE)" + - name: Cache dependencies + uses: actions/cache@v2.1.5 + with: + path: ${{ steps.go-cache-paths.outputs.go-build }} + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.OS }}-build-${{ env.cache-name }} + ${{ runner.OS }}-build- + ${{ runner.OS }}- - name: Build binary run: make build - name: Run E2E Tests env: E2E_KICS_BINARY: ${{ github.workspace }}/bin/kics run: | - env go test "github.com/Checkmarx/kics/e2e" -v From 7cdffc4d343078294bbbf4e8c28b760846a9d451 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rog=C3=A9rio=20Peixoto?= Date: Fri, 16 Apr 2021 14:35:26 +0100 Subject: [PATCH 05/13] fixing pipeline for windows --- .github/workflows/go-e2e.yaml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/go-e2e.yaml b/.github/workflows/go-e2e.yaml index 1b2d759afd1..b47ab3257ee 100644 --- a/.github/workflows/go-e2e.yaml +++ b/.github/workflows/go-e2e.yaml @@ -44,8 +44,16 @@ jobs: ${{ runner.OS }}- - name: Build binary run: make build + - name: Get Binary Path + id: getbin + run: | + #!/usr/bin/env python3 + import os + path = os.path.join(os.environ['GITHUB_WORKSPACE'], 'bin', 'kics') + print(f"::set-output name=kics::{path}") + shell: python {0} - name: Run E2E Tests env: - E2E_KICS_BINARY: ${{ github.workspace }}/bin/kics + E2E_KICS_BINARY: ${{ steps.getbin.outputs.kics }} run: | go test "github.com/Checkmarx/kics/e2e" -v From f538f0bde1f1f38ea60114d6befffed0cf214d08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rog=C3=A9rio=20Peixoto?= Date: Fri, 16 Apr 2021 14:53:23 +0100 Subject: [PATCH 06/13] fixing e2e pipeline --- .github/workflows/go-e2e.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/go-e2e.yaml b/.github/workflows/go-e2e.yaml index b47ab3257ee..b1155e16009 100644 --- a/.github/workflows/go-e2e.yaml +++ b/.github/workflows/go-e2e.yaml @@ -44,6 +44,10 @@ jobs: ${{ runner.OS }}- - name: Build binary run: make build + - run: | + set +o pipefail + python --version + python3 --version || true - name: Get Binary Path id: getbin run: | @@ -51,7 +55,7 @@ jobs: import os path = os.path.join(os.environ['GITHUB_WORKSPACE'], 'bin', 'kics') print(f"::set-output name=kics::{path}") - shell: python {0} + shell: python3 {0} - name: Run E2E Tests env: E2E_KICS_BINARY: ${{ steps.getbin.outputs.kics }} From 14c4c15626cae326613031191e97ba212fda93ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rog=C3=A9rio=20Peixoto?= Date: Fri, 16 Apr 2021 15:01:39 +0100 Subject: [PATCH 07/13] fixing e2e pipeline for windows --- .github/workflows/go-e2e.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/go-e2e.yaml b/.github/workflows/go-e2e.yaml index b1155e16009..2a611547768 100644 --- a/.github/workflows/go-e2e.yaml +++ b/.github/workflows/go-e2e.yaml @@ -45,9 +45,8 @@ jobs: - name: Build binary run: make build - run: | - set +o pipefail - python --version - python3 --version || true + ls + ls bin - name: Get Binary Path id: getbin run: | From f419c68e81519e1ea083f04272b4dc154c33b50c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rog=C3=A9rio=20Peixoto?= Date: Fri, 16 Apr 2021 15:54:22 +0100 Subject: [PATCH 08/13] e2e tests shall run only on ubuntu for now --- .github/workflows/go-e2e.yaml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/go-e2e.yaml b/.github/workflows/go-e2e.yaml index 2a611547768..f1c6a1c2590 100644 --- a/.github/workflows/go-e2e.yaml +++ b/.github/workflows/go-e2e.yaml @@ -11,7 +11,7 @@ jobs: strategy: matrix: go-version: [1.16.x] - os: [ubuntu-latest, windows-latest, macos-latest] + os: [ubuntu-latest] runs-on: ${{ matrix.os }} steps: - name: Cancel Previous Runs @@ -44,9 +44,6 @@ jobs: ${{ runner.OS }}- - name: Build binary run: make build - - run: | - ls - ls bin - name: Get Binary Path id: getbin run: | From eff214ad5e8475f316fd9d99050c05768db4ef44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Reigota?= Date: Fri, 16 Apr 2021 18:06:31 +0100 Subject: [PATCH 09/13] Enhanced E2E testing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit needs fixing Signed-off-by: João Reigota --- e2e/cli_test.go | 181 ++++++++++++++++++++++++++++-- e2e/fixtures/E2E_CLI_001 | 2 +- e2e/fixtures/E2E_CLI_002 | 39 +++++++ e2e/fixtures/E2E_CLI_003 | 39 +++++++ e2e/fixtures/E2E_CLI_004 | 40 +++++++ e2e/fixtures/E2E_CLI_005 | 0 e2e/fixtures/E2E_CLI_005_PAYLOAD | 29 +++++ e2e/fixtures/samples/terraform.tf | 18 +++ e2e/utils.go | 27 ++++- go.sum | 2 - payload.json | 29 +++++ 11 files changed, 386 insertions(+), 20 deletions(-) create mode 100644 e2e/fixtures/E2E_CLI_002 create mode 100644 e2e/fixtures/E2E_CLI_003 create mode 100644 e2e/fixtures/E2E_CLI_004 create mode 100644 e2e/fixtures/E2E_CLI_005 create mode 100644 e2e/fixtures/E2E_CLI_005_PAYLOAD create mode 100644 e2e/fixtures/samples/terraform.tf create mode 100755 payload.json diff --git a/e2e/cli_test.go b/e2e/cli_test.go index 21fc4f9e15a..a56d1acc8ff 100644 --- a/e2e/cli_test.go +++ b/e2e/cli_test.go @@ -1,24 +1,183 @@ package e2e import ( + "encoding/json" "fmt" + "reflect" "strings" "testing" "github.com/stretchr/testify/require" ) -// E2E_CLI_001 - KICS command should display a help text in the CLI when provided with the --help flag and it should describe the available commands plus the global flags -func Test_E2E_CLI_001(t *testing.T) { - kicsPath := getKICSBinaryPath("") - actualOutput, err := runCommandAndReturnOutput([]string{kicsPath, "--help"}) - require.NoError(t, err, "Capture output should not yield an error") - actualLines := strings.Split(actualOutput, "\n") +type logMsg struct { + Level string `json:"level"` + ErrorMgs string `json:"error"` + Message string `json:"message"` +} - expectedOutput, err := readFixture("E2E_CLI_001") - require.NoError(t, err, "Reading a fixture should not yield an error") - expectedLines := strings.Split(expectedOutput, "\n") - for idx := range expectedLines { - require.Equal(t, expectedLines[idx], actualLines[idx], fmt.Sprintf("Expected output line\n%s is not equal to actual output line\n%s\n line: %d", expectedLines[idx], actualLines[idx], idx)) +type cmdArgs []string + +type args struct { + args []cmdArgs // args to pass to kics binary + expectedOut []string // path to file with expected output + expectedPayload []string +} + +var tests = []struct { + name string + args args + wantStatus int + removePayload []string +}{ + // E2E_CLI_001 - KICS command should display a help text in the CLI when provided with the + // --help flag and it should describe the available commands plus the global flags + { + name: "E2E_CLI_001", + args: args{ + args: []cmdArgs{ + []string{"--help"}, + }, + expectedOut: []string{"E2E_CLI_001"}, + expectedPayload: []string{}, + }, + removePayload: []string{}, + wantStatus: 0, + }, + // E2E-CLI-002 - KICS scan command should display a help text in the CLI when provided with the + // --help flag and it should describe the options related with scan plus the global options + { + name: "E2E-CLI-002", + args: args{ + args: []cmdArgs{ + []string{"scan", "--help"}, + }, + expectedOut: []string{"E2E_CLI_002"}, + }, + wantStatus: 0, + }, + // E2E-CLI-003 - KICS scan command had a mandatory flag -p the CLI should exhibit + // an error message and return exit code 1 + { + name: "E2E-CLI-003", + args: args{ + args: []cmdArgs{ + []string{"scan"}, + }, + expectedOut: []string{"E2E_CLI_003"}, + }, + wantStatus: 1, + }, + // E2E-CLI-004 - KICS scan command had a mandatory flag -p the CLI should exhibit + // an error message and return exit code 1 + { + name: "E2E-CLI-004", + args: args{ + args: []cmdArgs{ + []string{"--ci", "--verbose"}, + []string{"scan", "--ci", "--verbose"}, + []string{"--ci", "scan", "--verbose"}, + }, + expectedOut: []string{ + "E2E_CLI_004", + "E2E_CLI_004", + "E2E_CLI_004", + }, + }, + wantStatus: 1, + }, + // E2E-CLI-005 - KICS scan with -- payload-path flag should create a file with the + // passed name containing the payload of the files scanned + { + name: "E2E-CLI-005", + args: args{ + args: []cmdArgs{ + []string{"scan", "--silent", "-p", "fixtures/samples/terraform.tf", + "--payload-path", "fixtures/payload.json"}, + }, + expectedOut: []string{ + "E2E_CLI_005", + }, + expectedPayload: []string{ + "E2E_CLI_005_PAYLOAD", + }, + }, + wantStatus: 0, + removePayload: []string{"payload.json"}, + }, +} + +func Test_E2E_CLI(t *testing.T) { + kicsPath := getKICSBinaryPath("/usr/local/bin/kics") + + for _, tt := range tests { + for arg := range tt.args.args { + t.Run(fmt.Sprintf("%s_%d", tt.name, arg), func(t *testing.T) { + out, err := runCommand(append(kicsPath, tt.args.args[arg]...)) + // Check command Error + require.NoError(t, err, "Capture output should not yield an error") + // Check exit status code + if !reflect.DeepEqual(out.status, tt.wantStatus) { + t.Errorf("kics status = %v, want status = %v", out.status, tt.wantStatus) + } + // Get and preapare expected output + want, err := prepareExpected(tt.args.expectedOut[arg]) + require.NoError(t, err, "Reading a fixture should not yield an error") + // Check Number of Lines + require.Equal(t, len(want), len(out.output), "\nExpected number of lines:%d\nKics number of lines:%d\n", len(want), len(out.output)) + // Check output lines + for idx := range want { + checkLine(t, out.output[idx], want[idx], idx+1) + } + // Check payload files + for _, file := range tt.removePayload { + fileCheck(t, file, tt.args.expectedPayload[arg]) + } + }) + } + } +} + +func prepareExpected(path string) ([]string, error) { + cont, err := readFixture(path) + if err != nil { + return []string{}, err } + return strings.Split(cont, "\n"), nil +} + +func checkLine(t *testing.T, expec, want string, line int) { + logExp := logMsg{} + logWant := logMsg{} + errE := json.Unmarshal([]byte(expec), &logExp) + errW := json.Unmarshal([]byte(want), &logWant) + if errE == nil && errW == nil { + checkJSONLog(t, logExp, logWant) + } else { + require.Equal(t, expec, want, + "\nExpected Output line\n%s\nKICS Output line:\n%s\n line: %d", want, expec, line) + } +} + +func checkJSONLog(t *testing.T, expec, want logMsg) { + require.Equal(t, expec.Level, want.Level, + "\nExpected Output line log level\n%s\nKICS Output line log level:\n%s\n", want.Level, expec.Level) + require.Equal(t, expec.ErrorMgs, want.ErrorMgs, + "\nExpected Output line error msg\n%s\nKICS Output line error msg:\n%s\n", expec.ErrorMgs, want.ErrorMgs) + require.Equal(t, expec.Message, want.Message, + "\nExpected Output line msg\n%s\nKICS Output line msg:\n%s\n", expec.Message, want.Message) +} + +func fileCheck(t *testing.T, remove, payload string) { + wantPayload, err := prepareExpected(payload) + require.NoError(t, err, "Reading a fixture should not yield an error") + expectPayload, err := prepareExpected(remove) + require.NoError(t, err, "Reading a fixture should not yield an error") + require.Equal(t, len(wantPayload), len(expectPayload), + "\nExpected file number of lines:%d\nKics file number of lines:%d\n", len(wantPayload), len(expectPayload)) + checkJSONFile(t, wantPayload, expectPayload) +} + +func checkJSONFile(t *testing.T, expect, want []string) { // Needs to fixed + require.Equal(t, expect, want) } diff --git a/e2e/fixtures/E2E_CLI_001 b/e2e/fixtures/E2E_CLI_001 index 36982dc2acc..3a630d7f83c 100644 --- a/e2e/fixtures/E2E_CLI_001 +++ b/e2e/fixtures/E2E_CLI_001 @@ -20,4 +20,4 @@ Flags: -s, --silent silence stdout messages (mutually exclusive with verbose and ci) -v, --verbose write logs to stdout too (mutually exclusive with silent) -Use "kics [command] --help" for more information about a command. \ No newline at end of file +Use "kics [command] --help" for more information about a command. diff --git a/e2e/fixtures/E2E_CLI_002 b/e2e/fixtures/E2E_CLI_002 new file mode 100644 index 00000000000..447cedb221d --- /dev/null +++ b/e2e/fixtures/E2E_CLI_002 @@ -0,0 +1,39 @@ +Executes a scan analysis + +Usage: + kics scan [flags] + +Flags: + --config string path to configuration file + --exclude-categories strings exclude categories by providing its name + can be provided multiple times or as a comma separated string + example: 'Access control,Best practices' + -e, --exclude-paths strings exclude paths from scan + supports glob and can be provided multiple times or as a quoted comma separated string + example: './shouldNotScan/*,somefile.txt' + --exclude-queries strings exclude queries by providing the query ID + can be provided multiple times or as a comma separated string + example: 'e69890e6-fce5-461d-98ad-cb98318dfc96,4728cd65-a20c-49da-8b31-9c08b423e4db' + -x, --exclude-results strings exclude results by providing the similarity ID of a result + can be provided multiple times or as a comma separated string + example: 'fec62a97d569662093dbb9739360942f...,31263s5696620s93dbb973d9360942fc2a...' + -h, --help help for scan + --minimal-ui simplified version of CLI output + --no-progress hides the progress bar + -o, --output-path string directory path to store reports + -p, --path string path or directory path to scan + -d, --payload-path string path to store internal representation JSON file + --preview-lines int number of lines to be display in CLI results (min: 1, max: 30) (default 3) + -q, --queries-path string path to directory with queries (default "./assets/queries") + --report-formats strings formats in which the results will be exported (json, sarif, html) + -t, --type strings case insensitive list of platform types to scan + (Ansible, CloudFormation, Dockerfile, Kubernetes, OpenAPI, Terraform) + +Global Flags: + --ci display only log messages to CLI output (mutually exclusive with silent) + -f, --log-format string determines log format (pretty,json) (default "pretty") + --log-level string determines log level (TRACE,DEBUG,INFO,WARN,ERROR,FATAL) (default "INFO") + --log-path string path to log files, (defaults to ${PWD}/info.log) + --no-color disable CLI color output + -s, --silent silence stdout messages (mutually exclusive with verbose and ci) + -v, --verbose write logs to stdout too (mutually exclusive with silent) diff --git a/e2e/fixtures/E2E_CLI_003 b/e2e/fixtures/E2E_CLI_003 new file mode 100644 index 00000000000..49a577393a9 --- /dev/null +++ b/e2e/fixtures/E2E_CLI_003 @@ -0,0 +1,39 @@ +Error: required flag(s) "path" not set +Usage: + kics scan [flags] + +Flags: + --config string path to configuration file + --exclude-categories strings exclude categories by providing its name + can be provided multiple times or as a comma separated string + example: 'Access control,Best practices' + -e, --exclude-paths strings exclude paths from scan + supports glob and can be provided multiple times or as a quoted comma separated string + example: './shouldNotScan/*,somefile.txt' + --exclude-queries strings exclude queries by providing the query ID + can be provided multiple times or as a comma separated string + example: 'e69890e6-fce5-461d-98ad-cb98318dfc96,4728cd65-a20c-49da-8b31-9c08b423e4db' + -x, --exclude-results strings exclude results by providing the similarity ID of a result + can be provided multiple times or as a comma separated string + example: 'fec62a97d569662093dbb9739360942f...,31263s5696620s93dbb973d9360942fc2a...' + -h, --help help for scan + --minimal-ui simplified version of CLI output + --no-progress hides the progress bar + -o, --output-path string directory path to store reports + -p, --path string path or directory path to scan + -d, --payload-path string path to store internal representation JSON file + --preview-lines int number of lines to be display in CLI results (min: 1, max: 30) (default 3) + -q, --queries-path string path to directory with queries (default "./assets/queries") + --report-formats strings formats in which the results will be exported (json, sarif, html) + -t, --type strings case insensitive list of platform types to scan + (Ansible, CloudFormation, Dockerfile, Kubernetes, OpenAPI, Terraform) + +Global Flags: + --ci display only log messages to CLI output (mutually exclusive with silent) + -f, --log-format string determines log format (pretty,json) (default "pretty") + --log-level string determines log level (TRACE,DEBUG,INFO,WARN,ERROR,FATAL) (default "INFO") + --log-path string path to log files, (defaults to ${PWD}/info.log) + --no-color disable CLI color output + -s, --silent silence stdout messages (mutually exclusive with verbose and ci) + -v, --verbose write logs to stdout too (mutually exclusive with silent) + diff --git a/e2e/fixtures/E2E_CLI_004 b/e2e/fixtures/E2E_CLI_004 new file mode 100644 index 00000000000..53cfe7f5607 --- /dev/null +++ b/e2e/fixtures/E2E_CLI_004 @@ -0,0 +1,40 @@ +Error: can't provide 'verbose' and 'ci' flags simultaneously +Usage: + kics scan [flags] + +Flags: + --config string path to configuration file + --exclude-categories strings exclude categories by providing its name + can be provided multiple times or as a comma separated string + example: 'Access control,Best practices' + -e, --exclude-paths strings exclude paths from scan + supports glob and can be provided multiple times or as a quoted comma separated string + example: './shouldNotScan/*,somefile.txt' + --exclude-queries strings exclude queries by providing the query ID + can be provided multiple times or as a comma separated string + example: 'e69890e6-fce5-461d-98ad-cb98318dfc96,4728cd65-a20c-49da-8b31-9c08b423e4db' + -x, --exclude-results strings exclude results by providing the similarity ID of a result + can be provided multiple times or as a comma separated string + example: 'fec62a97d569662093dbb9739360942f...,31263s5696620s93dbb973d9360942fc2a...' + -h, --help help for scan + --minimal-ui simplified version of CLI output + --no-progress hides the progress bar + -o, --output-path string directory path to store reports + -p, --path string path or directory path to scan + -d, --payload-path string path to store internal representation JSON file + --preview-lines int number of lines to be display in CLI results (min: 1, max: 30) (default 3) + -q, --queries-path string path to directory with queries (default "./assets/queries") + --report-formats strings formats in which the results will be exported (json, sarif, html) + -t, --type strings case insensitive list of platform types to scan + (Ansible, CloudFormation, Dockerfile, Kubernetes, OpenAPI, Terraform) + +Global Flags: + --ci display only log messages to CLI output (mutually exclusive with silent) + -f, --log-format string determines log format (pretty,json) (default "pretty") + --log-level string determines log level (TRACE,DEBUG,INFO,WARN,ERROR,FATAL) (default "INFO") + --log-path string path to log files, (defaults to ${PWD}/info.log) + --no-color disable CLI color output + -s, --silent silence stdout messages (mutually exclusive with verbose and ci) + -v, --verbose write logs to stdout too (mutually exclusive with silent) + +{"level":"error","error":"can't provide 'verbose' and 'ci' flags simultaneously","time":"2021-04-16T15:41:12+01:00","message":"Failed to run application"} diff --git a/e2e/fixtures/E2E_CLI_005 b/e2e/fixtures/E2E_CLI_005 new file mode 100644 index 00000000000..e69de29bb2d diff --git a/e2e/fixtures/E2E_CLI_005_PAYLOAD b/e2e/fixtures/E2E_CLI_005_PAYLOAD new file mode 100644 index 00000000000..ec4abb5b2f1 --- /dev/null +++ b/e2e/fixtures/E2E_CLI_005_PAYLOAD @@ -0,0 +1,29 @@ +{ + "document": [ + { + "id": "42f99d44-5a8c-4986-9d63-bf5baad91166", + "file": "kics/test/fixtures/tc-sim01/positive1.tf", + "resource": { + "aws_redshift_cluster": { + "default": { + "node_type": "dc1.large", + "cluster_type": "single-node", + "cluster_identifier": "tf-redshift-cluster", + "database_name": "mydb", + "master_username": "foo", + "master_password": "Mustbe8characters" + }, + "default1": { + "master_password": "Mustbe8characters", + "node_type": "dc1.large", + "cluster_type": "single-node", + "publicly_accessible": true, + "cluster_identifier": "tf-redshift-cluster", + "database_name": "mydb", + "master_username": "foo" + } + } + } + } + ] +} diff --git a/e2e/fixtures/samples/terraform.tf b/e2e/fixtures/samples/terraform.tf new file mode 100644 index 00000000000..d749ce1bd1f --- /dev/null +++ b/e2e/fixtures/samples/terraform.tf @@ -0,0 +1,18 @@ +resource "aws_redshift_cluster" "default" { + cluster_identifier = "tf-redshift-cluster" + database_name = "mydb" + master_username = "foo" + master_password = "Mustbe8characters" + node_type = "dc1.large" + cluster_type = "single-node" +} + +resource "aws_redshift_cluster" "default1" { + cluster_identifier = "tf-redshift-cluster" + database_name = "mydb" + master_username = "foo" + master_password = "Mustbe8characters" + node_type = "dc1.large" + cluster_type = "single-node" + publicly_accessible = true +} diff --git a/e2e/utils.go b/e2e/utils.go index 447af055508..4f4293e45d4 100644 --- a/e2e/utils.go +++ b/e2e/utils.go @@ -5,15 +5,30 @@ import ( "os" "os/exec" "path/filepath" + "strings" ) -func runCommandAndReturnOutput(args []string) (stdout string, err error) { +type cmdOutput struct { + output []string + status int +} + +func runCommand(args []string) (*cmdOutput, error) { cmd := exec.Command(args[0], args[1:]...) //nolint - stdOutput, err := cmd.Output() + stdOutput, err := cmd.CombinedOutput() if err != nil { - return "", err + if exitError, ok := err.(*exec.ExitError); ok { + return &cmdOutput{ + output: strings.Split(string(stdOutput), "\n"), + status: exitError.ExitCode(), + }, nil + } + return &cmdOutput{}, err } - return string(stdOutput), nil + return &cmdOutput{ + output: strings.Split(string(stdOutput), "\n"), + status: 0, + }, nil } func readFixture(testName string) (string, error) { @@ -32,12 +47,12 @@ func readFile(path string) (string, error) { return string(bytes), nil } -func getKICSBinaryPath(path string) string { +func getKICSBinaryPath(path string) []string { var rtnPath string if path == "" { rtnPath = os.Getenv("E2E_KICS_BINARY") } else { rtnPath = path } - return rtnPath + return []string{rtnPath} } diff --git a/go.sum b/go.sum index 4f75f723024..09627dcd7af 100644 --- a/go.sum +++ b/go.sum @@ -1335,7 +1335,6 @@ golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1588,7 +1587,6 @@ golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200616133436-c1934b75d054/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/payload.json b/payload.json new file mode 100755 index 00000000000..057e34695d6 --- /dev/null +++ b/payload.json @@ -0,0 +1,29 @@ +{ + "document": [ + { + "resource": { + "aws_redshift_cluster": { + "default": { + "node_type": "dc1.large", + "cluster_type": "single-node", + "cluster_identifier": "tf-redshift-cluster", + "database_name": "mydb", + "master_username": "foo", + "master_password": "Mustbe8characters" + }, + "default1": { + "database_name": "mydb", + "master_username": "foo", + "master_password": "Mustbe8characters", + "node_type": "dc1.large", + "cluster_type": "single-node", + "publicly_accessible": true, + "cluster_identifier": "tf-redshift-cluster" + } + } + }, + "id": "e0b25b59-9543-45b3-986b-59bef03b72c5", + "file": "/home/reigota/kics/test/fixtures/tc-sim01/positive1.tf" + } + ] +} From 6b34e4b17da4b18e5df98c039fa432d18722619c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Reigota?= Date: Mon, 19 Apr 2021 11:07:59 +0100 Subject: [PATCH 10/13] Added Checking to payload files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: João Reigota --- e2e/cli_test.go | 35 +++++++++++++++++++++++++++++++- e2e/fixtures/E2E_CLI_005_PAYLOAD | 4 ++-- 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/e2e/cli_test.go b/e2e/cli_test.go index a56d1acc8ff..a45cac0c576 100644 --- a/e2e/cli_test.go +++ b/e2e/cli_test.go @@ -3,10 +3,14 @@ package e2e import ( "encoding/json" "fmt" + "os" + "path/filepath" "reflect" "strings" "testing" + "github.com/Checkmarx/kics/pkg/model" + "github.com/Checkmarx/kics/test" "github.com/stretchr/testify/require" ) @@ -176,8 +180,37 @@ func fileCheck(t *testing.T, remove, payload string) { require.Equal(t, len(wantPayload), len(expectPayload), "\nExpected file number of lines:%d\nKics file number of lines:%d\n", len(wantPayload), len(expectPayload)) checkJSONFile(t, wantPayload, expectPayload) + err = os.Remove(filepath.Join("fixtures", remove)) + require.NoError(t, err) } func checkJSONFile(t *testing.T, expect, want []string) { // Needs to fixed - require.Equal(t, expect, want) + var wantI model.Documents + var expecI model.Documents + errE := json.Unmarshal([]byte(strings.Join(expect, "\n")), &expecI) + require.NoError(t, errE, "Unmarshaling JSON file should not yield an error") + errW := json.Unmarshal([]byte(strings.Join(want, "\n")), &wantI) + require.NoError(t, errW, "Unmarshaling JSON file should not yield an error") + setFields(t, wantI, expecI, "payload") +} + +func setFields(t *testing.T, want, expect model.Documents, location string) { + switch location { + case "payload": + for _, docs := range want.Documents { + require.NotNil(t, docs["id"]) // Here additional checks may be added as length of id, or contains in file + require.NotNil(t, docs["file"]) + docs["id"] = "0" + docs["file"] = "file" + } + if !reflect.DeepEqual(expect, want) { + expectStr, err := test.StringifyStruct(expect) + require.NoError(t, err) + wantStr, err := test.StringifyStruct(want) + require.NoError(t, err) + t.Errorf("Expected:\n%v\n,want:\n%v\n", expectStr, wantStr) + } + case "result": // TODO + default: + } } diff --git a/e2e/fixtures/E2E_CLI_005_PAYLOAD b/e2e/fixtures/E2E_CLI_005_PAYLOAD index ec4abb5b2f1..27ef0d0797a 100644 --- a/e2e/fixtures/E2E_CLI_005_PAYLOAD +++ b/e2e/fixtures/E2E_CLI_005_PAYLOAD @@ -1,8 +1,8 @@ { "document": [ { - "id": "42f99d44-5a8c-4986-9d63-bf5baad91166", - "file": "kics/test/fixtures/tc-sim01/positive1.tf", + "id": "0", + "file": "file", "resource": { "aws_redshift_cluster": { "default": { From 4eebf010df51e2950ae813bb76af67f1d3f51356 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Reigota?= Date: Mon, 19 Apr 2021 11:13:30 +0100 Subject: [PATCH 11/13] Cleared Path for Pipeline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: João Reigota --- e2e/cli_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/cli_test.go b/e2e/cli_test.go index a45cac0c576..d8918c425e4 100644 --- a/e2e/cli_test.go +++ b/e2e/cli_test.go @@ -112,7 +112,7 @@ var tests = []struct { } func Test_E2E_CLI(t *testing.T) { - kicsPath := getKICSBinaryPath("/usr/local/bin/kics") + kicsPath := getKICSBinaryPath("") for _, tt := range tests { for arg := range tt.args.args { From 39fefff3bf6b5e3af3a190de2046c684213b4d3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rog=C3=A9rio=20Peixoto?= Date: Mon, 19 Apr 2021 12:11:18 +0100 Subject: [PATCH 12/13] fixing e2e tests --- .github/workflows/go-e2e.yaml | 3 +++ e2e/cli_test.go | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/go-e2e.yaml b/.github/workflows/go-e2e.yaml index 20cd1dcec14..82a43b778e2 100644 --- a/.github/workflows/go-e2e.yaml +++ b/.github/workflows/go-e2e.yaml @@ -50,9 +50,12 @@ jobs: import os path = os.path.join(os.environ['GITHUB_WORKSPACE'], 'bin', 'kics') print(f"::set-output name=kics::{path}") + queries_path = os.path.join(os.environ['GITHUB_WORKSPACE'], 'assets', 'queries') + print(f"::set-output name=queries::{queries_path}") shell: python3 {0} - name: Run E2E Tests env: E2E_KICS_BINARY: ${{ steps.getbin.outputs.kics }} + E2E_KICS_QUERIES_PATH: ${{ steps.getbin.outputs.queries }} run: | go test "github.com/Checkmarx/kics/e2e" -v diff --git a/e2e/cli_test.go b/e2e/cli_test.go index d8918c425e4..1b9773f20e6 100644 --- a/e2e/cli_test.go +++ b/e2e/cli_test.go @@ -96,7 +96,7 @@ var tests = []struct { name: "E2E-CLI-005", args: args{ args: []cmdArgs{ - []string{"scan", "--silent", "-p", "fixtures/samples/terraform.tf", + []string{"scan", "--silent", "-q", "../assets/queries", "-p", "fixtures/samples/terraform.tf", "--payload-path", "fixtures/payload.json"}, }, expectedOut: []string{ @@ -128,7 +128,7 @@ func Test_E2E_CLI(t *testing.T) { want, err := prepareExpected(tt.args.expectedOut[arg]) require.NoError(t, err, "Reading a fixture should not yield an error") // Check Number of Lines - require.Equal(t, len(want), len(out.output), "\nExpected number of lines:%d\nKics number of lines:%d\n", len(want), len(out.output)) + require.Equal(t, len(want), len(out.output), "\nExpected number of stdout lines:%d\nActual of stdout lines:%d\n", len(want), len(out.output)) // Check output lines for idx := range want { checkLine(t, out.output[idx], want[idx], idx+1) From df2881eaea03f287d1c0cb7bb2e4bb06fdb3254b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Reigota?= Date: Mon, 19 Apr 2021 12:27:19 +0100 Subject: [PATCH 13/13] Fix e2e testing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: João Reigota --- e2e/cli_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/e2e/cli_test.go b/e2e/cli_test.go index 1b9773f20e6..e6feba3279c 100644 --- a/e2e/cli_test.go +++ b/e2e/cli_test.go @@ -97,7 +97,7 @@ var tests = []struct { args: args{ args: []cmdArgs{ []string{"scan", "--silent", "-q", "../assets/queries", "-p", "fixtures/samples/terraform.tf", - "--payload-path", "fixtures/payload.json"}, + "--payload-path", "fixtures/payload.json", "-q", "../assets/queries"}, }, expectedOut: []string{ "E2E_CLI_005", @@ -128,7 +128,8 @@ func Test_E2E_CLI(t *testing.T) { want, err := prepareExpected(tt.args.expectedOut[arg]) require.NoError(t, err, "Reading a fixture should not yield an error") // Check Number of Lines - require.Equal(t, len(want), len(out.output), "\nExpected number of stdout lines:%d\nActual of stdout lines:%d\n", len(want), len(out.output)) + require.Equal(t, len(want), len(out.output), + "\nExpected number of stdout lines:%d\nActual of stdout lines:%d\n", len(want), len(out.output)) // Check output lines for idx := range want { checkLine(t, out.output[idx], want[idx], idx+1)