Skip to content

Commit

Permalink
adds: nightly build and separate e2e for vulnerability
Browse files Browse the repository at this point in the history
  • Loading branch information
Rchanger committed Aug 27, 2021
1 parent 5ab68e2 commit 9ac6fea
Show file tree
Hide file tree
Showing 10 changed files with 211 additions and 128 deletions.
39 changes: 0 additions & 39 deletions .github/workflows/e2e-vuln.yml

This file was deleted.

10 changes: 7 additions & 3 deletions .github/workflows/gobuild.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@ name: build
on:
push:
pull_request:
schedule:
- cron: 0 23 * * *
jobs:
validate:
runs-on: ubuntu-latest
Expand All @@ -13,11 +15,9 @@ jobs:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID_TEST }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY_TEST }}
AWS_REGION: ${{ secrets.AWS_REGION_TEST }}
AZURE_AUTH_TEST_SECRET: ${{ secrets.AZURE_AUTH_TEST_SECRET }}
AZURE_AUTH_TEST_SECRET: ${{ secrets.AZURE_AUTH_TEST_KEY }}
GOOGLE_APPLICATION_CREDENTIALS_TEST_SECRET: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS_TEST_KEY }}
steps:
- name: check event types
run: echo ${{ github.event_name }}
- name: Checkout Terrascan
uses: actions/checkout@v2

Expand All @@ -44,6 +44,10 @@ jobs:
- name: Run e2e tests
run: make e2e-tests

- name: Run e2e vulnerability tests
if: ${{ github.event_name == 'push'|| github.event_name == 'schedule' }}
run: make e2e-vulnerability-tests

- name: Upload coverage to Codecov
uses: codecov/codecov-action@v1

Expand Down
8 changes: 5 additions & 3 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ help:
@echo "unit-tests\n\texecute unit tests"
@echo "e2e-tests\n\texecute e2e tests"
@echo "e2e-admission-control-tests\n\texecute e2e admission control tests"
@echo "e2e-vulnerability-tests\n\texecute e2e vulnerability tests"
@echo "validate\n\trun all validations"

# build terrascan binary
Expand Down Expand Up @@ -95,6 +96,10 @@ e2e-tests: build
e2e-admission-control-tests: build
./scripts/e2e-admission-control.sh

# run e2e vulnerability tests
e2e-vulnerability-tests: build
./scripts/e2e-vulnerability.sh

# install kind
install-kind:
./scripts/install-kind.sh
Expand Down Expand Up @@ -132,6 +137,3 @@ atlantis-docker-push-latest:
# push release tag terrascan_atlantis docker image
atlantis-docker-push-latest-tag:
./scripts/atlantis/docker-push-latest-tag.sh

e2e-vuln-tests: build
./scripts/run-e2e-vuln.sh
2 changes: 1 addition & 1 deletion scripts/run-e2e-vuln.sh → scripts/e2e-vulnerability.sh
Original file line number Diff line number Diff line change
Expand Up @@ -6,4 +6,4 @@ set -o pipefail

export TERRASCAN_BIN_PATH=${PWD}/bin/terrascan

go test -p 1 -v ./test/e2e/scan/...
go test -p 1 -v ./test/e2e/vulnerability/...
2 changes: 1 addition & 1 deletion scripts/run-e2e.sh
Original file line number Diff line number Diff line change
Expand Up @@ -6,4 +6,4 @@ set -o pipefail

export TERRASCAN_BIN_PATH=${PWD}/bin/terrascan

go test -p 1 -v ./test/...
go test -p 1 -v $(go list ./test/e2e/... | grep -v /vulnerability)
62 changes: 0 additions & 62 deletions test/e2e/scan/scan_rules_filtering_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@
package scan_test

import (
"io/ioutil"
"os"
"path/filepath"

Expand All @@ -35,13 +34,6 @@ var (
categoryIncorrectError string = "category not supported"
)

const (
googleApplicationKeyTestEnvName = "GOOGLE_APPLICATION_CREDENTIALS_TEST_SECRET"
googleApplicationKeyEnvName = "GOOGLE_APPLICATION_CREDENTIALS"
azureAuthKeyTestEnvName = "AZURE_AUTH_TEST_SECRET"
azureAuthKeyEnvName = "AZURE_AUTH_LOCATION"
)

var _ = Describe("Scan command with rule filtering options", func() {

BeforeEach(func() {
Expand Down Expand Up @@ -387,58 +379,4 @@ var _ = Describe("Scan command with rule filtering options", func() {
})
})
})
Describe("resource with conatiner images to be scanned for vulnerabilities", func() {
resourceVulnerabilityIacRelPath := filepath.Join(iacRootRelPath, "resource_for_vulnerability_scan")
Context("resource with aws ecr registry image used which has vulnerabilities", func() {
iacDir := filepath.Join(resourceVulnerabilityIacRelPath, "aws_ecr_registry_used_in_resource")
It("should display vulnerabilities and exit with status code 0 since no violations found", func() {
scanArgs := []string{"-p", policyDir, "-i", "terraform", "-d", iacDir, "-o", "json", "--find-vuln"}
scanUtils.RunScanAndVerifyVulnerabilityOutputCount(terrascanBinaryPath, helper.ExitCodeZero, false, true, outWriter, errWriter, scanArgs...)
})
})
Context("resource with azure registry image used which has vulnerabilities", func() {
iacDir := filepath.Join(resourceVulnerabilityIacRelPath, "azure_registry_used_in_resource")
var tempFile *os.File
JustBeforeEach(func() {
data := os.Getenv(azureAuthKeyTestEnvName)
if data != "" {
tempFile, err := ioutil.TempFile("", "azure.auth")
if err != nil {
Expect(err).NotTo(HaveOccurred())
}
tempFile.Write([]byte(data))
os.Setenv(azureAuthKeyEnvName, tempFile.Name())
}
})
JustAfterEach(func() {
os.Remove(tempFile.Name())
})
It("should display vulnerabilities and exit with status code 0 since no violations found", func() {
scanArgs := []string{"-p", policyDir, "-i", "terraform", "-d", iacDir, "-o", "json", "--find-vuln"}
scanUtils.RunScanAndVerifyVulnerabilityOutputCount(terrascanBinaryPath, helper.ExitCodeZero, false, true, outWriter, errWriter, scanArgs...)
})
})
Context("resource with google registry image used which has vulnerabilities", func() {
iacDir := filepath.Join(resourceVulnerabilityIacRelPath, "google_gcr_registry_used_in_resource")
var tempFile *os.File
JustBeforeEach(func() {
data := os.Getenv(googleApplicationKeyTestEnvName)
if data != "" {
tempFile, err := ioutil.TempFile("", "app-key.json")
if err != nil {
Expect(err).NotTo(HaveOccurred())
}
tempFile.Write([]byte(data))
os.Setenv(googleApplicationKeyEnvName, tempFile.Name())
}
})
JustAfterEach(func() {
os.Remove(tempFile.Name())
})
It("should display vulnerabilities and exit with status code 0 since no violations found", func() {
scanArgs := []string{"-p", policyDir, "-i", "k8s", "-d", iacDir, "-o", "json", "--find-vuln"}
scanUtils.RunScanAndVerifyVulnerabilityOutputCount(terrascanBinaryPath, helper.ExitCodeZero, false, true, outWriter, errWriter, scanArgs...)
})
})
})
})
19 changes: 0 additions & 19 deletions test/e2e/scan/scan_utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@ package scan
import (
"io"
"path/filepath"
"time"

"github.com/accurics/terrascan/test/helper"
"github.com/onsi/gomega"
Expand All @@ -35,9 +34,6 @@ const (

// RemoteScanTimeout is default scan command remote execution timeout
RemoteScanTimeout int = 30

// VulnerabilityScanTimeout is default Vulnerability Scan timeout
VulnerabilityScanTimeout time.Duration = 2 * time.Minute
)

// RunScanAndAssertGoldenOutputRegex runs the scan command with supplied paramters and compares actual and golden output
Expand Down Expand Up @@ -104,18 +100,3 @@ func RunScanAndAssertGoldenSarifOutputRegex(terrascanBinaryPath, relGoldenFilePa
session, goldenFileAbsPath := RunScanCommand(terrascanBinaryPath, relGoldenFilePath, exitCode, outWriter, errWriter, args...)
helper.CompareActualSarifOutputWithGoldenSummaryRegex(session, goldenFileAbsPath)
}

// RunScanAndVerifyVulnerabilityOutputCount runs the scan command with supplied paramters and checks scan summary output
func RunScanAndVerifyVulnerabilityOutputCount(terrascanBinaryPath string, exitCode int, isJunitXML, isStdOut bool, outWriter, errWriter io.Writer, args ...string) {
session := RunScanCommandWithFindVulnerability(terrascanBinaryPath, exitCode, outWriter, errWriter, args...)
helper.CheckSummaryForVulnerabilities(session, isStdOut)
}

// RunScanCommandWithFindVulnerability with --find-vuln flag executes the scan command, validates exit code
func RunScanCommandWithFindVulnerability(terrascanBinaryPath string, exitCode int, outWriter, errWriter io.Writer, args ...string) *gexec.Session {
argList := []string{ScanCommand}
argList = append(argList, args...)
session := helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, argList...)
gomega.Eventually(session, VulnerabilityScanTimeout).Should(gexec.Exit(exitCode))
return session
}
116 changes: 116 additions & 0 deletions test/e2e/vulnerability/vulnerability_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,116 @@
/*
Copyright (C) 2020 Accurics, Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package vulnerability_test

import (
"io"
"io/ioutil"
"os"
"path/filepath"

vulnUtils "github.com/accurics/terrascan/test/e2e/vulnerability"
"github.com/accurics/terrascan/test/helper"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/onsi/gomega/gbytes"
)

var (
terrascanBinaryPath string
iacRootRelPath = filepath.Join("..", "test_data", "iac")
policyRootRelPath = filepath.Join("..", "test_data", "policies")
outWriter, errWriter io.Writer
tempFile *os.File
err error
)

const (
googleApplicationKeyTestEnvName = "GOOGLE_APPLICATION_CREDENTIALS_TEST_SECRET"
googleApplicationKeyEnvName = "GOOGLE_APPLICATION_CREDENTIALS"
azureAuthKeyTestEnvName = "AZURE_AUTH_TEST_SECRET"
azureAuthKeyEnvName = "AZURE_AUTH_LOCATION"
)

var _ = Describe("Vulnerabilty", func() {

BeforeSuite(func() {
terrascanBinaryPath = helper.GetTerrascanBinaryPath()
})

BeforeEach(func() {
outWriter = gbytes.NewBuffer()
errWriter = gbytes.NewBuffer()
})

AfterEach(func() {
outWriter = nil
errWriter = nil
})

Describe("resource with conatiner images to be scanned for vulnerabilities", func() {
resourceVulnerabilityIacRelPath := filepath.Join(iacRootRelPath, "resource_for_vulnerability_scan")
Context("resource with aws ecr registry image used which has vulnerabilities", func() {
iacDir := filepath.Join(resourceVulnerabilityIacRelPath, "aws_ecr_registry_used_in_resource")
It("should display vulnerabilities and exit with status code 0 since no violations found", func() {
scanArgs := []string{"-p", policyRootRelPath, "-i", "terraform", "-d", iacDir, "-o", "json"}
vulnUtils.RunScanAndVerifyVulnerabilityOutputCount(terrascanBinaryPath, helper.ExitCodeZero, false, true, outWriter, errWriter, scanArgs...)
})
})
Context("resource with azure registry image used which has vulnerabilities", func() {
iacDir := filepath.Join(resourceVulnerabilityIacRelPath, "azure_registry_used_in_resource")
JustBeforeEach(func() {
data := os.Getenv(azureAuthKeyTestEnvName)
if data != "" {
tempFile, err = ioutil.TempFile("", "azure.auth")
Expect(err).NotTo(HaveOccurred())
tempFile.Write([]byte(data))
os.Setenv(azureAuthKeyEnvName, tempFile.Name())
}
})
JustAfterEach(func() {
if tempFile != nil && tempFile.Name() != "" {
os.Remove(tempFile.Name())
}
})
It("should display vulnerabilities and exit with status code 0 since no violations found", func() {
scanArgs := []string{"-p", policyRootRelPath, "-i", "terraform", "-d", iacDir, "-o", "json"}
vulnUtils.RunScanAndVerifyVulnerabilityOutputCount(terrascanBinaryPath, helper.ExitCodeZero, false, true, outWriter, errWriter, scanArgs...)
})
})
Context("resource with google registry image used which has vulnerabilities", func() {
iacDir := filepath.Join(resourceVulnerabilityIacRelPath, "google_gcr_registry_used_in_resource")
JustBeforeEach(func() {
data := os.Getenv(googleApplicationKeyTestEnvName)
if data != "" {
tempFile, err = ioutil.TempFile("", "app-key.json")
Expect(err).NotTo(HaveOccurred())
tempFile.Write([]byte(data))
os.Setenv(googleApplicationKeyEnvName, tempFile.Name())
}
})
JustAfterEach(func() {
if tempFile != nil && tempFile.Name() != "" {
os.Remove(tempFile.Name())
}
})
It("should display vulnerabilities and exit with status code 0 since no violations found", func() {
scanArgs := []string{"-p", policyRootRelPath, "-i", "k8s", "-d", iacDir, "-o", "json"}
vulnUtils.RunScanAndVerifyVulnerabilityOutputCount(terrascanBinaryPath, helper.ExitCodeZero, false, true, outWriter, errWriter, scanArgs...)
})
})
})
})
29 changes: 29 additions & 0 deletions test/e2e/vulnerability/vulnerabilty_suit_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
/*
Copyright (C) 2020 Accurics, Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package vulnerability_test

import (
"testing"

. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)

func TestServer(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Vulnerability Fetch Suite")
}
Loading

0 comments on commit 9ac6fea

Please sign in to comment.