Skip to content

Commit

Permalink
feat(telemetry): opt-out telemetry and fixing sentry dsn provisioning
Browse files Browse the repository at this point in the history
closes #3041

Signed-off-by: Rogério Peixoto <[email protected]>
  • Loading branch information
rogeriopeixotocx committed Apr 30, 2021
1 parent bd4e347 commit 9eedd50
Show file tree
Hide file tree
Showing 20 changed files with 183 additions and 115 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/go-ci-coverage.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -69,5 +69,5 @@ jobs:
git add coverage.svg
git add coverage.html
git status
git commit -m 'chore: updating coverage'
git commit -m 'chore(tests): updating test coverage report and badge'
git push origin gh-pages
149 changes: 75 additions & 74 deletions .github/workflows/go-ci-integration.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,82 +9,83 @@ jobs:
name: integration-tests
runs-on: ubuntu-latest
steps:
- name: Cancel Previous Runs
uses: styfle/[email protected]
with:
access_token: ${{ github.token }}
- name: Check out code into the Go module directory
uses: actions/checkout@v2
with:
persist-credentials: false
- name: Set up Docker Buildx
id: buildx
uses: docker/setup-buildx-action@master
- name: Cache Docker layers
uses: actions/[email protected]
with:
path: /tmp/.buildx-cache
key: ${{ runner.os }}-buildx-${{ github.ref }}
restore-keys: |
${{ runner.os }}-buildx-${{ github.ref }}
- name: Get short SHA
run: echo "GITHUB_SHA_SHORT=$(echo $GITHUB_SHA | cut -c 1-8)" >> $GITHUB_ENV
- name: Build
id: docker_build
uses: docker/build-push-action@v2
with:
load: true
context: ./
file: ./Dockerfile
builder: ${{ steps.buildx.outputs.name }}
push: false
tags: kics:${{ github.sha }}
build-args: |
VERSION=${GITHUB_SHA_SHORT}
COMMIT=${GITHUB_SHA}
cache-from: type=local,src=/tmp/.buildx-cache
cache-to: type=local,dest=/tmp/.buildx-cache
- name: Image digest
run: echo ${{ steps.docker_build.outputs.digest }}
- name: Run docker image and generate results.json
run: |
docker run -v ${PWD}/assets/queries:/path \
-e SENTRY_DSN=${{secrets.SENTRY_DSN}} \
kics:${{ github.sha }} scan --silent --ignore-on-exit "results" --log-level DEBUG --log-file --log-path "/path/info.log" -p "/path" -o "/path/results.json"
- name: Archive test logs
uses: actions/upload-artifact@v2
if: always()
with:
name: integration-logs-${{ github.event.pull_request.head.sha }}
path: assets/queries/info.log
- name: Display results
run: |
cat ${PWD}/assets/queries/results.json
- name: Archive test results
uses: actions/upload-artifact@v2
with:
name: integration-results-${{ github.event.pull_request.head.sha }}
path: assets/queries/results.json
- name: Assert results.json
run: |
set -eo pipefail
- name: Cancel Previous Runs
uses: styfle/[email protected]
with:
access_token: ${{ github.token }}
- name: Check out code into the Go module directory
uses: actions/checkout@v2
with:
persist-credentials: false
- name: Set up Docker Buildx
id: buildx
uses: docker/setup-buildx-action@master
- name: Cache Docker layers
uses: actions/[email protected]
with:
path: /tmp/.buildx-cache
key: ${{ runner.os }}-buildx-${{ github.ref }}
restore-keys: |
${{ runner.os }}-buildx-${{ github.ref }}
- name: Get short SHA
run: echo "GITHUB_SHA_SHORT=$(echo $GITHUB_SHA | cut -c 1-8)" >> $GITHUB_ENV
- name: Build
id: docker_build
uses: docker/build-push-action@v2
with:
load: true
context: ./
file: ./Dockerfile
builder: ${{ steps.buildx.outputs.name }}
push: false
tags: kics:${{ github.sha }}
build-args: |
VERSION=${GITHUB_SHA_SHORT}
COMMIT=${GITHUB_SHA}
SENTRY_DSN=${{ secrets.SENTRY_DSN }}
cache-from: type=local,src=/tmp/.buildx-cache
cache-to: type=local,dest=/tmp/.buildx-cache
- name: Image digest
run: echo ${{ steps.docker_build.outputs.digest }}
- name: Run docker image and generate results.json
run: |
docker run -v ${PWD}/assets/queries:/path \
-e SENTRY_DSN=${{secrets.SENTRY_DSN}} \
kics:${{ github.sha }} scan --silent --ignore-on-exit "results" --log-level DEBUG --log-file --log-path "/path/info.log" -p "/path" -o "/path/results.json"
- name: Archive test logs
uses: actions/upload-artifact@v2
if: always()
with:
name: integration-logs-${{ github.event.pull_request.head.sha }}
path: assets/queries/info.log
- name: Display results
run: |
cat ${PWD}/assets/queries/results.json
- name: Archive test results
uses: actions/upload-artifact@v2
with:
name: integration-results-${{ github.event.pull_request.head.sha }}
path: assets/queries/results.json
- name: Assert results.json
run: |
set -eo pipefail
COUNT=$(jq '.queries_total' ${PWD}/assets/queries/results.json)
echo "Assert queries_total > 0 in results.json :: ${COUNT}"
echo $COUNT | xargs -i{} test {} -gt 0
COUNT=$(jq '.queries_total' ${PWD}/assets/queries/results.json)
echo "Assert queries_total > 0 in results.json :: ${COUNT}"
echo $COUNT | xargs -i{} test {} -gt 0
COUNT=$(jq '.total_counter' ${PWD}/assets/queries/results.json)
echo "Assert total_counter > 0 in results.json :: ${COUNT}"
echo $COUNT | xargs -i{} test {} -gt 0
COUNT=$(jq '.total_counter' ${PWD}/assets/queries/results.json)
echo "Assert total_counter > 0 in results.json :: ${COUNT}"
echo $COUNT | xargs -i{} test {} -gt 0
COUNT=$(jq '.files_scanned' ${PWD}/assets/queries/results.json)
echo "Assert files_scanned > 0 in results.json :: ${COUNT}"
echo $COUNT | xargs -i{} test {} -gt 0
COUNT=$(jq '.files_scanned' ${PWD}/assets/queries/results.json)
echo "Assert files_scanned > 0 in results.json :: ${COUNT}"
echo $COUNT | xargs -i{} test {} -gt 0
COUNT=$(jq '.queries_failed_to_execute' ${PWD}/assets/queries/results.json)
echo "Assert queries_failed_to_execute == 0 in results.json :: ${COUNT}"
echo $COUNT | xargs -i{} test {} -eq 0
COUNT=$(jq '.queries_failed_to_execute' ${PWD}/assets/queries/results.json)
echo "Assert queries_failed_to_execute == 0 in results.json :: ${COUNT}"
echo $COUNT | xargs -i{} test {} -eq 0
COUNT=$(jq '.files_failed_to_scan' ${PWD}/assets/queries/results.json)
echo "Assert files_failed_to_scan == 0 in results.json :: ${COUNT}"
echo $COUNT | xargs -i{} test {} -eq 0
COUNT=$(jq '.files_failed_to_scan' ${PWD}/assets/queries/results.json)
echo "Assert files_failed_to_scan == 0 in results.json :: ${COUNT}"
echo $COUNT | xargs -i{} test {} -eq 0
4 changes: 2 additions & 2 deletions .github/workflows/mkdocs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@ on:
branches:
- master
paths:
- 'docs/**'
- 'mkdocs.yml'
- "docs/**"
- "mkdocs.yml"
jobs:
deploy:
runs-on: ubuntu-latest
Expand Down
2 changes: 2 additions & 0 deletions .github/workflows/release-dkr-image-for-tag.yml
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ jobs:
build-args: |
VERSION=${{ github.event.inputs.tag }}
COMMIT=${{ github.sha }}
SENTRY_DSN=${{ secrets.SENTRY_DSN }}
- name: Push alpine to Docker Hub
if: ${{ hashFiles('Dockerfile.integration') }} != ""
uses: docker/build-push-action@v2
Expand All @@ -75,3 +76,4 @@ jobs:
build-args: |
VERSION=${{ github.event.inputs.tag }}
COMMIT=${{ github.sha }}
SENTRY_DSN=${{ secrets.SENTRY_DSN }}
2 changes: 2 additions & 0 deletions .github/workflows/release-dkr-image.yml
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ jobs:
build-args: |
VERSION=${{ steps.get-version.outputs.version }}
COMMIT=${{ github.sha }}
SENTRY_DSN=${{ secrets.SENTRY_DSN }}
- name: Push alpine to Docker Hub
uses: docker/build-push-action@v2
id: build_alpine
Expand All @@ -60,6 +61,7 @@ jobs:
build-args: |
VERSION=${{ steps.get-version.outputs.version }}
COMMIT=${{ github.sha }}
SENTRY_DSN=${{ secrets.SENTRY_DSN }}
- name: Update repo description
uses: peter-evans/dockerhub-description@v2
with:
Expand Down
1 change: 1 addition & 0 deletions .github/workflows/release-nightly.yml
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ jobs:
args: release --rm-dist --snapshot --skip-validate --config="./.goreleaser-nightly.yml"
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
- name: delete release
uses: dev-drprasad/[email protected]
with:
Expand Down
3 changes: 2 additions & 1 deletion .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ name: goreleaser
on:
push:
tags:
- 'v*'
- "v*"

jobs:
goreleaser:
Expand Down Expand Up @@ -39,3 +39,4 @@ jobs:
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
KICS_BOT_PAT: ${{ secrets.KICS_BOT_PAT }}
SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
4 changes: 2 additions & 2 deletions .github/workflows/update-docs-index.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -38,9 +38,9 @@ jobs:
- name: Create pull request
uses: peter-evans/create-pull-request@v3
with:
title: "docs(kicsbot): updating links in index page - ${{ steps.cversion.outputs.version }}"
title: "docs(index): updating links in index page - ${{ steps.cversion.outputs.version }}"
token: ${{ secrets.KICS_BOT_PAT }}
commit-message: "docs(kicsbot): updating links in index page"
commit-message: "docs(index): updating links in index page"
delete-branch: true
branch: feature/kicsbot-update-docs-index
base: master
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/update-docs-queries.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -33,9 +33,9 @@ jobs:
- name: Create Pull Request
uses: peter-evans/create-pull-request@v3
with:
title: "docs(kicsbot): update queries catalog"
title: "docs(queries): update queries catalog"
token: ${{ secrets.KICS_BOT_PAT }}
commit-message: "docs(kicsbot): update queries catalog"
commit-message: "docs(queries): update queries catalog"
delete-branch: true
branch: feature/kicsbot-update-queries-docs
body: |
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/update-install-script.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -54,10 +54,10 @@ jobs:
- name: Create Pull Request
uses: peter-evans/create-pull-request@v3
with:
title: "chore(kicsbot): update install script"
title: "chore(install): update install script"
token: ${{ secrets.KICS_BOT_PAT }}
delete-branch: true
commit-message: "chore(kicsbot): update install script"
commit-message: "chore(install): update install script"
branch: feature/kicsbot-update-install-script
base: master
body: |
Expand Down
7 changes: 5 additions & 2 deletions .goreleaser-nightly.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,11 @@ builds:
goarch:
- amd64
ldflags:
- -X github.com/Checkmarx/kics/internal/constants.Version={{.Version}}-{{.ShortCommit}}
- -X github.com/Checkmarx/kics/internal/constants.SCMCommit={{.Commit}}
- -s
- -w
- -X github.com/Checkmarx/kics/internal/constants.Version={{ .Version }}-{{ .ShortCommit }}
- -X github.com/Checkmarx/kics/internal/constants.SCMCommit={{ .Commit }}
- -X github.com/Checkmarx/kics/internal/constants.SentryDSN={{ .Env.SENTRY_DSN }}
archives:
- builds: [kics]
format_overrides:
Expand Down
9 changes: 6 additions & 3 deletions .goreleaser.yml
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,11 @@ builds:
- goos: windows
goarm: 7
ldflags:
- -X github.com/Checkmarx/kics/internal/constants.Version={{.Version}}
- -X github.com/Checkmarx/kics/internal/constants.SCMCommit={{.Commit}}
- -s
- -w
- -X github.com/Checkmarx/kics/internal/constants.Version={{ .Version }}
- -X github.com/Checkmarx/kics/internal/constants.SCMCommit={{ .Commit }}
- -X github.com/Checkmarx/kics/internal/constants.SentryDSN={{ .Env.SENTRY_DSN }}
archives:
- builds: [kics]
format_overrides:
Expand All @@ -39,7 +42,7 @@ brews:
name: homebrew-tap
token: "{{ .Env.KICS_BOT_PAT }}"
folder: Formula
homepage: https://github.com/Checkmarx/kics
homepage: https://github.com/Checkmarx/kics
description: Find security vulnerabilities, compliance issues, and infrastructure misconfigurations in your IaC
license: Apache
test: |
Expand Down
7 changes: 4 additions & 3 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,9 @@ USER Checkmarx
WORKDIR /app

ENV GOPRIVATE=github.com/Checkmarx/*
ARG VERSION="dev"
ARG COMMIT="N/A"
ARG VERSION="development"
ARG COMMIT="NOCOMMIT"
ARG SENTRY_DSN=""

#Copy go mod and sum files
COPY --chown=Checkmarx:Checkmarx go.mod .
Expand All @@ -24,7 +25,7 @@ COPY . .
USER root
# Build the Go app
RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build \
-ldflags "-X github.com/Checkmarx/kics/internal/constants.Version=${VERSION} -X github.com/Checkmarx/kics/internal/constants.SCMCommit=${COMMIT}" \
-ldflags "-s -w -X github.com/Checkmarx/kics/internal/constants.Version=${VERSION} -X github.com/Checkmarx/kics/internal/constants.SCMCommit=${COMMIT} -X github.com/Checkmarx/kics/internal/constants.SentryDSN=${SENTRY_DSN}" \
-a -installsuffix cgo \
-o bin/kics cmd/console/main.go
USER Checkmarx
Expand Down
6 changes: 4 additions & 2 deletions Dockerfile.integration
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,9 @@ WORKDIR /app


ENV GOPRIVATE=github.com/Checkmarx/*
ARG VERSION=development
ARG VERSION="development"
ARG COMMIT="NOCOMMIT"
ARG SENTRY_DSN=""

#Copy go mod and sum files
COPY --chown=Checkmarx:Checkmarx go.mod .
Expand All @@ -24,7 +26,7 @@ COPY . .
USER root
# Build the Go app
RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build \
-ldflags "-X github.com/Checkmarx/kics/internal/constants.Version=${VERSION}" -a -installsuffix cgo \
-ldflags "-s -w -X github.com/Checkmarx/kics/internal/constants.Version=${VERSION} -X github.com/Checkmarx/kics/internal/constants.SCMCommit=${COMMIT} -X github.com/Checkmarx/kics/internal/constants.SentryDSN=${SENTRY_DSN}" -a -installsuffix cgo \
-o bin/kics cmd/console/main.go
USER Checkmarx

Expand Down
6 changes: 3 additions & 3 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ COMMIT := $(shell git rev-parse HEAD)
VERSION := snapshot-$(shell echo ${COMMIT} | cut -c1-8)
IMAGE_TAG := dev
TARGET_BIN ?= bin/kics
CONSTANTS_PATH = github.com/Checkmarx/kics/internal/constants

.PHONY: clean
clean: ## remove files created during build
Expand Down Expand Up @@ -44,15 +45,14 @@ build-all: ## go build for both kics and query builder
build-all: lint generate
$(call print-target)
@go build -o bin/ \
-ldflags "-X github.com/Checkmarx/kics/internal/constants.Version=${VERSION} -X github.com/Checkmarx/kics/internal/constants.SCMCommit=${COMMIT}" ./...
-ldflags "-X ${CONSTANTS_PATH}.Version=${VERSION} -X ${CONSTANTS_PATH}.SCMCommit=${COMMIT}" ./...
@mv bin/console bin/kics

.PHONY: build
build: ## go build
build: generate
$(call print-target)
@go build -o ${TARGET_BIN} \
-ldflags "-X github.com/Checkmarx/kics/internal/constants.Version=${VERSION} -X github.com/Checkmarx/kics/internal/constants.SCMCommit=${COMMIT}" \
@go build -o ${TARGET_BIN} -ldflags "-X ${CONSTANTS_PATH}.SCMCommit=${COMMIT} -X ${CONSTANTS_PATH}.Version=${VERSION}" \
cmd/console/main.go

.PHONY: go-clean
Expand Down
13 changes: 12 additions & 1 deletion docs/usage/commands.md
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
# KICS Command
# KICS CLI

## KICS Command

KICS has the following commands available:

Expand Down Expand Up @@ -84,3 +86,12 @@ Global Flags:
The other commands have no further options.

---

## Disable Telemetry

You can disable KICS telemetry with `KICS_COLLECT_TELEMETRY` environment variable set to `0` or `false` e.g:

```sh
# 'KICS telemetry disabled' message should appear in stderr
KICS_COLLECT_TELEMETRY=0 ./bin/kics version
```
Loading

0 comments on commit 9eedd50

Please sign in to comment.