Merge branch 'master' into feature/add-kics-e2e-tests

.dockerignore

@@ -9,7 +9,6 @@ test
 .golangci.yml
 .goreleaser.nightly.yml
 .goreleaser.yml
-codecov.yml
 cx.configuration
 docker-compose.yml
 Dockerfile

.github/scripts/get-test-metrics.sh

@@ -0,0 +1,93 @@
+#!/usr/bin/env zsh
+
+list_cfn_samples() {
+  for sample in assets/queries/cloudFormation/**/test/{positive,negative}[0-9].{yaml,json}; do echo $sample; done
+}
+
+list_openapi_samples(){
+  for sample in assets/queries/openAPI/**/test/{positive,negative}[0-9]*.{yaml,json}; do echo $sample; done
+}
+
+list_common_samples(){
+  for sample in assets/queries/common/**/test/*[0-9].{tf,json,yaml,dockerfile}; do echo $sample; done
+}
+
+list_ansible_samples(){
+  for sample in assets/queries/ansible/**/test/*.yaml; do echo $sample; done
+}
+
+list_docker_samples(){
+  for sample in assets/queries/**/test/*.dockerfile; do echo $sample; done
+}
+
+list_terraform_samples(){
+  for sample in assets/queries/terraform/**/test/*.tf; do echo $sample; done
+}
+
+run_unit_tests_and_filter_subtests(){
+  go test ./... -v | grep -v TestQueriesContent/ | grep -v TestQueriesMetadata/ | grep -v TestQueries/ | grep PASS
+}
+
+println(){
+  printf "|%-25s| %7d|\n" $@
+}
+
+print_header(){
+  printf "|%-25s| %7s|\n" $@
+}
+
+print_sep(){
+  printf '|'
+  printf '-%.0s' {1..25}
+  printf '|'
+  printf '-%.0s' {1..8}
+  printf '|'
+  printf '\n'
+}
+
+echo "#################################"
+echo "#          TEST METRICS         #"
+echo "#################################"
+
+OPENAPI_SAMPLES=$(list_openapi_samples | wc -l)
+COMMON_SAMPLES=$(list_common_samples | wc -l)
+CFN_SAMPLES=$(list_cfn_samples | wc -l)
+ANSIBLE_SAMPLES=$(list_ansible_samples | wc -l)
+DKR_SAMPLES=$(list_docker_samples | wc -l)
+TF_SAMPLES=$(list_terraform_samples | wc -l)
+TOTAL_SAMPLES=$((${TF_SAMPLES} + ${DKR_SAMPLES} + ${ANSIBLE_SAMPLES} + ${CFN_SAMPLES} + ${COMMON_SAMPLES} + ${OPENAPI_SAMPLES}))
+
+echo "::group::Samples Metrics"
+print_sep
+print_header "Platform" "Samples"
+print_sep
+println "Ansible" "${ANSIBLE_SAMPLES}"
+println "CloudFormation" "${CFN_SAMPLES}"
+println "Common" "${COMMON_SAMPLES}"
+println "Docker" "${DKR_SAMPLES}"
+println "OpenAPI" "${OPENAPI_SAMPLES}"
+println "Terraform" "${TF_SAMPLES}"
+print_sep
+println "Total" "${TOTAL_SAMPLES}"
+print_sep
+echo "::endgroup::"
+
+echo "::set-output name=ansible::${ANSIBLE_SAMPLES}"
+echo "::set-output name=cfn::${CFN_SAMPLES}"
+echo "::set-output name=common::${COMMON_SAMPLES}"
+echo "::set-output name=docker::${DKR_SAMPLES}"
+echo "::set-output name=openapi::${OPENAPI_SAMPLES}"
+echo "::set-output name=terraform::${TF_SAMPLES}"
+echo
+echo "Install Test Dependencies"
+echo "::group::Install Test Dependencis"
+go mod vendor
+echo "::endgroup::"
+echo
+echo "Running Unit Tests..."
+echo "::group::Unit Tests Metrics"
+TOTAL_TESTS=$(run_unit_tests_and_filter_subtests | wc -l)
+echo "Total unit tests: ${TOTAL_TESTS}"
+echo "::endgroup::"
+echo
+echo "::set-output name=total_tests::${TOTAL_TESTS}"

.github/workflows/go-ci.yml

@@ -60,17 +60,18 @@ jobs:
         persist-credentials: false
     - name: Run Go mod tidy
       run: go mod tidy
+    - name: Get cache paths
+      id: go-cache-paths
+      run: |
+        echo "::set-output name=go-build::$(go env GOCACHE)"
+        echo "::set-output name=go-mod::$(go env GOMODCACHE)"
     - name: Cache dependencies
       uses: actions/[email protected]
       with:
-        path: |
-          ~/go/pkg/mod
-          ~/.cache/go-build
-          ~/Library/Caches/go-build
-          %LocalAppData%\go-build
+        path: ${{ steps.go-cache-paths.outputs.go-build }}
         key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
         restore-keys: |
-          ${{ runner.OS }}-build-${{ env.cache-name }}-
+          ${{ runner.OS }}-build-${{ env.cache-name }}
           ${{ runner.OS }}-build-
           ${{ runner.OS }}-
     - name: Get go-junit-report module
@@ -83,30 +84,14 @@ jobs:
         go mod vendor
     - name: Test and Generate Report
       run: |
+        set +o pipefail
         go test -mod=vendor -v $(go list ./... | grep -v e2e/) -count=1 -coverprofile cover.out 2>&1 | go-junit-report -set-exit-code -go-version ${{ matrix.go-version }} -package-name "github.com/Checkmarx/kics/test" > test-report-${{ matrix.os }}.xml
     - name: Archive unit tests report
       uses: actions/upload-artifact@v2
       with:
         name: unit-tests-report-${{ matrix.os }}-${{ github.event.pull_request.head.sha }}
         path: |
           test-report*.xml
-    - name: CodeCov
-      if: matrix.os == 'ubuntu-latest'
-      run: |
-        bash <(curl -s https://codecov.io/bash)
-    - name: Check if total coverage is greater then 0
-      if: matrix.os == 'ubuntu-latest'
-      run: |
-        CODE_COV=$(go tool cover -func cover.out | grep total | awk '{print substr($3, 1, length($3)-1)}')
-        EXPECTED_CODE_COV=0
-        var=$(awk 'BEGIN{ print "'$CODE_COV'"<"'$EXPECTED_CODE_COV'" }')
-        if [ "$var" -eq 1 ];then
-          echo "Your code coverage is too low. Coverage precentage is: $CODE_COV"
-          exit 1
-        else
-          echo "Your code coverage test passed! Coverage precentage is: $CODE_COV"
-          exit 0
-        fi
   security-scan:
     name: security-scan
     runs-on: ubuntu-latest
@@ -123,3 +108,17 @@ jobs:
         uses: github/codeql-action/upload-sarif@v1
         with:
           sarif_file: results.sarif
+  metrics:
+    name: test-metrics
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Source
+        uses: actions/checkout@v2
+      - name: Set up Go 1.x
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.16.x
+      - name: Install zsh
+        run: sudo apt install zsh
+      - name: Run test metrics script
+        run: zsh .github/scripts/get-test-metrics.sh

.github/workflows/release-dkr-image-for-tag.yml

@@ -0,0 +1,58 @@
+name: release-manual-docker-tag
+
+on:
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: 'Git Tag'
+        required: true
+
+jobs:
+  push_to_registry:
+    name: Push Docker image to Docker Hub
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v2
+        with:
+          ref: ${{ github.event.inputs.tag }}
+      - uses: toko-bifrost/ms-teams-deploy-card@master
+        if: always()
+        with:
+          github-token: ${{ secrets.KICS_BOT_PAT }}
+          webhook-uri: ${{ secrets.MSTEAMS_WEBHOOK_URL }}
+          card-layout-start: cozy
+          card-layout-exit: complete
+          show-on-start: true
+          show-on-exit: true
+          custom-actions: |
+            - name: View CI Logs
+              value: https://github.com/Checkmarx/kics/actions/runs/${{ github.run_id }}
+            - name: View HEAD Commit
+              value: https://github.com/Checkmarx/kics/commit/${{ github.sha }}
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      - name: Push scratch to Docker Hub
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: checkmarx/kics:latest,checkmarx/kics:${{ github.event.inputs.tag }}
+          build-args: |
+            VERSION=${{ github.event.inputs.tag }}
+            COMMIT=${{ github.sha }}
+      - name: Push alpine to Docker Hub
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: ./Dockerfile.integration
+          push: true
+          tags: checkmarx/kics:latest-alpine,checkmarx/kics:${{ github.event.inputs.tag }}-alpine
+          build-args: |
+            VERSION=${{ github.event.inputs.tag }}
+            COMMIT=${{ github.sha }}

.github/workflows/release-dkr-image.yml

@@ -29,7 +29,7 @@ jobs:
           show-on-exit: true
           custom-actions: |
             - name: View CI Logs
-              value: https://github.com/Checkmarx/kics/actions/runs/${{ github.run_id }}"
+              value: https://github.com/Checkmarx/kics/actions/runs/${{ github.run_id }}
             - name: View HEAD Commit
               value: https://github.com/Checkmarx/kics/commit/${{ github.sha }}
       - name: Set up Docker Buildx

.github/workflows/release-nightly.yml

@@ -18,7 +18,7 @@ jobs:
           fetch-depth: 0
       - name: Check if there are new commits since last nightly
         id: lasttag
-        run: echo "::set-output name=newchanges::$(bash ./.github/new_changes_nightly.sh)"
+        run: echo "::set-output name=newchanges::$(bash ./.github/scripts/new-changes-nightly.sh)"
       - name: Set short hash
         id: shorthash
         run: echo "::set-output name=sha8::$(echo ${GITHUB_SHA} | cut -c1-8)"

README.md

@@ -2,7 +2,6 @@
 [![Go Report Card](https://goreportcard.com/badge/github.com/Checkmarx/kics)](https://goreportcard.com/report/github.com/Checkmarx/kics)
 [![Codacy Badge](https://app.codacy.com/project/badge/Grade/ceddb5b1b37d4edfa56440842c6248a4)](https://www.codacy.com/gh/Checkmarx/kics/dashboard?utm_source=github.com&utm_medium=referral&utm_content=Checkmarx/kics&utm_campaign=Badge_Grade)
 [![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=Checkmarx_kics&metric=alert_status)](https://sonarcloud.io/dashboard?id=Checkmarx_kics)
-[![codecov](https://codecov.io/gh/Checkmarx/kics/branch/master/graph/badge.svg?token=SN0NO4H46G)](https://codecov.io/gh/Checkmarx/kics)
 [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
 [![Latest Release](https://img.shields.io/github/v/release/checkmarx/kics)](https://github.com/checkmarx/kics/releases)
 ![Docker Pulls](https://img.shields.io/docker/pulls/checkmarx/kics)

assets/libraries/openapi/library.rego

@@ -1,6 +1,6 @@
 package generic.openapi
 
-checkOpenAPI(doc) = version {
+check_openapi(doc) = version {
   object.get(doc, "openapi", "undefined") != "undefined"
   version = doc.openapi
   regex.match("^3\\.0\\.\\d+$", doc.openapi)

assets/queries/openAPI/global_server_uses_http/metadata.json

@@ -0,0 +1,9 @@
+{
+  "id": "2d8c175a-6d90-412b-8b0e-e034ea49a1fe",
+  "queryName": "Global Server Object Uses HTTP",
+  "severity": "MEDIUM",
+  "category": "Encryption",
+  "descriptionText": "Global server object URL should use 'https' protocol instead of 'http'",
+  "descriptionUrl": "https://swagger.io/specification/#server-object",
+  "platform": "OpenAPI"
+}

assets/queries/openAPI/global_server_uses_http/query.rego

@@ -0,0 +1,36 @@
+package Cx
+
+import data.generic.openapi as openAPILib
+
+CxPolicy[result] {
+	doc := input.document[i]
+	openAPILib.check_openapi(doc) != "undefined"
+    object.get(doc, "servers", "undefined") == "undefined"
+
+	result := {
+		"documentId": doc.id,
+		"searchKey": "openapi",
+		"issueType": "MissingAttribute",
+		"keyExpectedValue": "Global servers array should be defined",
+		"keyActualValue": "Global servers array is not defined",
+	}
+}
+
+CxPolicy[result] {
+	doc := input.document[i]
+	openAPILib.check_openapi(doc) != "undefined"
+	object.get(doc, "servers", "undefined") != "undefined"
+
+    count(doc.servers) > 0
+    object.get(doc.servers[j], "url", "undefined") != "undefined"
+    serverObj := doc.servers[j]
+    not startswith(serverObj.url, "https")
+
+	result := {
+		"documentId": doc.id,
+		"searchKey": sprintf("servers.url.%s", [serverObj.url]),
+		"issueType": "IncorrectValue",
+		"keyExpectedValue": "Global servers' URL should use HTTPS protocol",
+		"keyActualValue": "Global servers' URL are not using HTTPS protocol",
+	}
+}