Adds: e2e tests for vulnerability

.github/workflows/gobuild.yml

@@ -2,6 +2,8 @@ name: build
 on:
   push:
   pull_request:
+  schedule:
+    - cron: 0 23 * * *
 jobs:
   validate:
     runs-on: ubuntu-latest
@@ -10,10 +12,15 @@ jobs:
       GOPATH: /home/runner/work/terrascan
       GOBIN: /home/runner/work/terrascan/bin
       GO_VERSION: 1.16
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID_TEST }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY_TEST }}
+      AWS_REGION: ${{ secrets.AWS_REGION_TEST }}
+      AZURE_AUTH_TEST_SECRET:  ${{ secrets.AZURE_AUTH_TEST_KEY }}
+      GOOGLE_APPLICATION_CREDENTIALS_TEST_SECRET:  ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS_TEST_KEY }}
     steps:
     - name: Checkout Terrascan
       uses: actions/checkout@v2
-
+      
     - name: Setup Go
       uses: actions/setup-go@v1
       with:
@@ -37,6 +44,10 @@ jobs:
     - name: Run e2e tests
       run: make e2e-tests
 
+    - name: Run e2e vulnerability tests
+      if: ${{ github.event_name == 'push'|| github.event_name == 'schedule' }}
+      run: make e2e-vulnerability-tests
+
     - name: Upload coverage to Codecov
       uses: codecov/codecov-action@v1
 

Makefile

@@ -31,6 +31,7 @@ help:
 	@echo "unit-tests\n\texecute unit tests"
 	@echo "e2e-tests\n\texecute e2e tests"
 	@echo "e2e-admission-control-tests\n\texecute e2e admission control tests"
+	@echo "e2e-vulnerability-tests\n\texecute e2e vulnerability tests"
 	@echo "validate\n\trun all validations"
 
 # build terrascan binary
@@ -95,6 +96,10 @@ e2e-tests: build
 e2e-admission-control-tests: build
 	./scripts/e2e-admission-control.sh
 
+# run e2e vulnerability tests
+e2e-vulnerability-tests: build
+	./scripts/e2e-vulnerability.sh
+
 # install kind
 install-kind:
 	./scripts/install-kind.sh

scripts/e2e-vulnerability.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+export TERRASCAN_BIN_PATH=${PWD}/bin/terrascan
+
+go test -p 1 -v ./test/e2e/vulnerability/...

scripts/run-e2e.sh

@@ -6,4 +6,4 @@ set -o pipefail
 
 export TERRASCAN_BIN_PATH=${PWD}/bin/terrascan
 
-go test -p 1 -v ./test/...
+go test -p 1 -v $(go list ./test/e2e/... | grep -v /vulnerability) 

test/e2e/test_data/iac/resource_for_vulnerability_scan/aws_ecr_registry_used_in_resource/main.tf

@@ -0,0 +1,37 @@
+resource "aws_ecs_task_definition" "demo-ecs-task-definition" {
+  family                   = "ecs-task-definition-demo"
+  network_mode             = "awsvpc"
+  requires_compatibilities = ["FARGATE"]
+  memory                   = "1024"
+  cpu                      = "512"
+  execution_role_arn       = "arn:aws:iam::123456789012:role/ecsTaskExecutionRole"
+  container_definitions    =  <<TASK_DEFINITION
+[
+    {
+        "cpu": 10,
+        "command": ["sleep", "10"],
+        "entryPoint": ["/"],
+        "environment": [
+            {"name": "VARNAME", "value": "VARVAL"}
+        ],
+        "essential": true,
+        "image": "245578940568.dkr.ecr.us-east-2.amazonaws.com/terrascan-e2e-test:terrascan",
+        "memory": 128,
+        "name": "jenkins",
+        "portMappings": [
+            {
+                "containerPort": 80,
+                "hostPort": 8080
+            }
+        ],
+        "resourceRequirements":[
+            {
+                "type":"InferenceAccelerator",
+                "value":"device_1"
+            }
+        ]
+    }
+]
+TASK_DEFINITION
+
+}

test/e2e/test_data/iac/resource_for_vulnerability_scan/azure_registry_used_in_resource/main.tf

@@ -0,0 +1,20 @@
+resource "azurerm_container_group" "example" {
+  name                = "example-continst"
+  location            = azurerm_resource_group.example.location
+  resource_group_name = azurerm_resource_group.example.name
+  ip_address_type     = "public"
+  dns_name_label      = "aci-label"
+  os_type             = "Linux"
+
+  container {
+    name   = "terrascan"
+    image  = "terrascan.azurecr.io/terrascan-e2e-test:nginx@sha256:3f13b4376446cf92b0cb9a5c46ba75d57c41f627c4edb8b635fa47386ea29e20"
+    cpu    = "0.5"
+    memory = "1.5"
+
+    ports {
+      port     = 443
+      protocol = "TCP"
+    }
+  }
+}

test/e2e/test_data/iac/resource_for_vulnerability_scan/google_gcr_registry_used_in_resource/deployment.yaml

@@ -0,0 +1,31 @@
+apiVersion: v1
+kind: Pod
+metadata: 
+    name: backend-server
+    labels:
+        tier: backend
+        app: goserver
+spec:
+    securityContext:
+       seccompProfile:
+         type: RuntimeDefault
+    containers:
+        - name: goserver
+          image: gcr.io/accurics/terrascan-e2e-test:terrascan@sha256:a7185142bbef5ccad92fc9bca0e09a522c2d163efcaa472fd31987d68ff9c61d
+          securityContext:
+            readOnlyRootFilesystem: true
+            runAsNonRoot: true   
+          resources:
+            requests:
+                cpu: "1"
+                memory: "256Mi"
+            limits:
+                cpu: "1"
+                memory: "256Mi" 
+          readinessProbe:
+            exec:
+            command:
+            - go 
+            - version
+            periodSeconds: 10
+            initialDelaySeconds: 10

test/e2e/vulnerability/vulnerability_test.go

@@ -0,0 +1,119 @@
+/*
+    Copyright (C) 2020 Accurics, Inc.
+
+	Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+		http://www.apache.org/licenses/LICENSE-2.0
+
+	Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+*/
+
+package vulnerability_test
+
+import (
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	vulnUtils "github.com/accurics/terrascan/test/e2e/vulnerability"
+	"github.com/accurics/terrascan/test/helper"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/onsi/gomega/gbytes"
+)
+
+var (
+	terrascanBinaryPath  string
+	iacRootRelPath       = filepath.Join("..", "test_data", "iac")
+	policyRootRelPath    = filepath.Join("..", "test_data", "policies")
+	outWriter, errWriter io.Writer
+	tempFile             *os.File
+	err                  error
+)
+
+const (
+	googleApplicationKeyTestEnvName = "GOOGLE_APPLICATION_CREDENTIALS_TEST_SECRET"
+	googleApplicationKeyEnvName     = "GOOGLE_APPLICATION_CREDENTIALS"
+	azureAuthKeyTestEnvName         = "AZURE_AUTH_TEST_SECRET"
+	azureAuthKeyEnvName             = "AZURE_AUTH_LOCATION"
+)
+
+var _ = Describe("Vulnerabilty", func() {
+
+	BeforeSuite(func() {
+		terrascanBinaryPath = helper.GetTerrascanBinaryPath()
+	})
+
+	BeforeEach(func() {
+		outWriter = gbytes.NewBuffer()
+		errWriter = gbytes.NewBuffer()
+	})
+
+	AfterEach(func() {
+		outWriter = nil
+		errWriter = nil
+	})
+
+	Describe("resource with conatiner images to be scanned for vulnerabilities", func() {
+		resourceVulnerabilityIacRelPath := filepath.Join(iacRootRelPath, "resource_for_vulnerability_scan")
+		Context("resource with aws ecr registry image used which has vulnerabilities", func() {
+			iacDir := filepath.Join(resourceVulnerabilityIacRelPath, "aws_ecr_registry_used_in_resource")
+			It("should display vulnerabilities and vulnerability count should be present in scan summary", func() {
+				scanArgs := []string{"-p", policyRootRelPath, "-i", "terraform", "-d", iacDir, "-o", "json"}
+				// exit code is zero since no violations are present for resource
+				vulnUtils.RunScanAndVerifyVulnerabilityCount(terrascanBinaryPath, helper.ExitCodeZero, 1, outWriter, errWriter, scanArgs...)
+			})
+		})
+		Context("resource with azure registry image used which has vulnerabilities", func() {
+			iacDir := filepath.Join(resourceVulnerabilityIacRelPath, "azure_registry_used_in_resource")
+			JustBeforeEach(func() {
+				data := os.Getenv(azureAuthKeyTestEnvName)
+				if data != "" {
+					tempFile, err = ioutil.TempFile("", "azure.auth")
+					Expect(err).NotTo(HaveOccurred())
+					tempFile.Write([]byte(data))
+					os.Setenv(azureAuthKeyEnvName, tempFile.Name())
+				}
+			})
+			JustAfterEach(func() {
+				if tempFile != nil && tempFile.Name() != "" {
+					os.Remove(tempFile.Name())
+				}
+			})
+			It("should display vulnerabilities and  vulnerability count should be present in scan summary", func() {
+				scanArgs := []string{"-p", policyRootRelPath, "-i", "terraform", "-d", iacDir, "-o", "json"}
+				// exit code is zero since no violations are present for resource
+				vulnUtils.RunScanAndVerifyVulnerabilityCount(terrascanBinaryPath, helper.ExitCodeZero, 1, outWriter, errWriter, scanArgs...)
+			})
+		})
+		Context("resource with google registry image used which has vulnerabilities", func() {
+			iacDir := filepath.Join(resourceVulnerabilityIacRelPath, "google_gcr_registry_used_in_resource")
+			JustBeforeEach(func() {
+				data := os.Getenv(googleApplicationKeyTestEnvName)
+				if data != "" {
+					tempFile, err = ioutil.TempFile("", "app-key.json")
+					Expect(err).NotTo(HaveOccurred())
+					tempFile.Write([]byte(data))
+					os.Setenv(googleApplicationKeyEnvName, tempFile.Name())
+				}
+			})
+			JustAfterEach(func() {
+				if tempFile != nil && tempFile.Name() != "" {
+					os.Remove(tempFile.Name())
+				}
+			})
+			It("should display vulnerabilities and  vulnerability count should be present in scan summary", func() {
+				scanArgs := []string{"-p", policyRootRelPath, "-i", "k8s", "-d", iacDir, "-o", "json"}
+				// exit code is zero since no violations are present for resource
+				vulnUtils.RunScanAndVerifyVulnerabilityCount(terrascanBinaryPath, helper.ExitCodeZero, 1, outWriter, errWriter, scanArgs...)
+			})
+		})
+	})
+})

test/e2e/vulnerability/vulnerabilty_suite_test.go

@@ -0,0 +1,29 @@
+/*
+    Copyright (C) 2020 Accurics, Inc.
+
+	Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+		http://www.apache.org/licenses/LICENSE-2.0
+
+	Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+*/
+
+package vulnerability_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestServer(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Vulnerability Fetch Suite")
+}

test/e2e/vulnerability/vulnerabilty_utils.go

@@ -0,0 +1,52 @@
+/*
+    Copyright (C) 2020 Accurics, Inc.
+
+	Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+		http://www.apache.org/licenses/LICENSE-2.0
+
+	Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+*/
+
+package vulnerability
+
+import (
+	"io"
+	"time"
+
+	"github.com/accurics/terrascan/test/helper"
+	"github.com/onsi/gomega"
+	"github.com/onsi/gomega/gexec"
+)
+
+const (
+	// ScanCommand is terrascan's scan command
+	ScanCommand string = "scan"
+
+	// FindVulnerabilityFlag is terrascan's flag used with scan command
+	FindVulnerabilityFlag string = "--find-vuln"
+
+	// VulnerabilityScanTimeout is default Vulnerability Scan timeout
+	VulnerabilityScanTimeout time.Duration = 2 * time.Minute
+)
+
+// RunScanAndVerifyVulnerabilityCount runs the scan command with supplied paramters and checks scan summary output
+func RunScanAndVerifyVulnerabilityCount(terrascanBinaryPath string, exitCode, expectedCount int, outWriter, errWriter io.Writer, args ...string) {
+	session := RunScanCommandWithFindVulnerability(terrascanBinaryPath, exitCode, outWriter, errWriter, args...)
+	helper.CheckSummaryForVulnerabilities(session, expectedCount)
+}
+
+// RunScanCommandWithFindVulnerability with --find-vuln flag executes the scan command, validates exit code
+func RunScanCommandWithFindVulnerability(terrascanBinaryPath string, exitCode int, outWriter, errWriter io.Writer, args ...string) *gexec.Session {
+	argList := []string{ScanCommand, FindVulnerabilityFlag}
+	argList = append(argList, args...)
+	session := helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, argList...)
+	gomega.Eventually(session, VulnerabilityScanTimeout).Should(gexec.Exit(exitCode))
+	return session
+}

test/helper/helper.go

@@ -426,3 +426,24 @@ func CompareActualSarifOutputWithGoldenSummaryRegex(session *gexec.Session, gold
 
 	gomega.Expect(sessionOutput).Should(gomega.BeIdenticalTo(fileContents))
 }
+
+// CheckSummaryForVulnerabilities is a helper function to check vulnerabilies exists
+func CheckSummaryForVulnerabilities(session *gexec.Session, expectedCount int) {
+	var sessionBytes []byte
+
+	sessionBytes = session.Wait().Out.Contents()
+
+	sessionBytes = bytes.TrimSpace(sessionBytes)
+
+	var sessionEngineOutput policy.EngineOutput
+
+	err := json.Unmarshal(sessionBytes, &sessionEngineOutput)
+	gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+	sessionOutputSummary := sessionEngineOutput.ViolationStore.Summary
+	gomega.Expect(sessionOutputSummary).NotTo(gomega.BeNil())
+	gomega.Expect(sessionOutputSummary.Vulnerabilities).NotTo(gomega.BeNil())
+	gomega.Eventually(func() int {
+		return *sessionOutputSummary.Vulnerabilities
+	}).Should(gomega.BeNumerically(">=", expectedCount))
+}