diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..95c91bd4 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,19 @@ +*.egg-info +*.py[co] +*.sw[op] + +/.git +/.coverage +/.github +/.pytest_cache +/.python-version +/.tox +/.vscode +/tmp +/user-config +/venv +/venv* + +.*ignore +!.gitignore +!.dockerignore diff --git a/.gitignore b/.gitignore index 487c3491..addba6d8 100644 --- a/.gitignore +++ b/.gitignore @@ -10,5 +10,6 @@ .*ignore !.gitignore +!.dockerignore .python-version .vscode diff --git a/.secrets.baseline b/.secrets.baseline index f4f5d592..62db3dfc 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -3,7 +3,7 @@ "files": "test_data/.*|tests/.*|^.secrets.baseline$", "lines": null }, - "generated_at": "2020-01-13T18:12:22Z", + "generated_at": "2020-02-17T15:57:42Z", "plugins_used": [ { "name": "AWSKeyDetector" @@ -204,15 +204,6 @@ "type": "Hex High Entropy String" } ], - "test_data/short_files/first_line.py": [ - { - "hashed_secret": "0de9a11b3f37872868ca49ecd726c955e25b6e21", - "is_secret": false, - "is_verified": false, - "line_number": 1, - "type": "Hex High Entropy String" - } - ], "test_data/short_files/last_line.ini": [ { "hashed_secret": "0de9a11b3f37872868ca49ecd726c955e25b6e21", diff --git a/.travis.yml b/.travis.yml index 9f686fa1..bb3e3ce6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,12 +5,14 @@ env: - TOXENV=py27 - DOCKER_REGISTRY=us.icr.io - DOCKER_USER=iamapikey + - DOCKER_LOCAL_IMAGE=git-defenders/dsl - DOCKER_IMAGE=us.icr.io/git-defenders/dsl - DOCKER_IMAGE_TAG="$TRAVIS_BUILD_NUMBER-id-$TRAVIS_BUILD_ID-time-$(date +%s)" - DOCKER_IMAGE_TAG_DSS="$TRAVIS_BRANCH-$TRAVIS_BUILD_NUMBER-id-$TRAVIS_BUILD_ID-time-$(date +%s)" deploy: - provider: script script: echo ${IBM_CLOUD_API_KEY} | docker login -u $DOCKER_USER --password-stdin $DOCKER_REGISTRY + && docker tag $DOCKER_LOCAL_IMAGE $DOCKER_IMAGE:$DOCKER_IMAGE_TAG && docker push $DOCKER_IMAGE:$DOCKER_IMAGE_TAG && docker tag $DOCKER_IMAGE:$DOCKER_IMAGE_TAG $DOCKER_IMAGE:latest && docker push $DOCKER_IMAGE:latest @@ -18,7 +20,7 @@ deploy: branch: master - provider: script script: echo ${IBM_CLOUD_API_KEY} | docker login -u $DOCKER_USER --password-stdin $DOCKER_REGISTRY - && docker tag $DOCKER_IMAGE:$DOCKER_IMAGE_TAG $DOCKER_IMAGE:$DOCKER_IMAGE_TAG_DSS + && docker tag $DOCKER_LOCAL_IMAGE $DOCKER_IMAGE:$DOCKER_IMAGE_TAG_DSS && docker push $DOCKER_IMAGE:$DOCKER_IMAGE_TAG_DSS && docker tag $DOCKER_IMAGE:$DOCKER_IMAGE_TAG_DSS $DOCKER_IMAGE:dss-latest && docker push $DOCKER_IMAGE:dss-latest @@ -39,7 +41,7 @@ matrix: # python: pypy install: - pip install tox -script: make test && docker build -t $DOCKER_IMAGE:$DOCKER_IMAGE_TAG --no-cache . +script: make test && ./build-dockerfiles.sh cache: directories: - $HOME/.cache/pre-commit diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index aa83d9a1..00000000 --- a/Dockerfile +++ /dev/null @@ -1,11 +0,0 @@ -FROM python:alpine -MAINTAINER toolbox-dev@us.ibm.com -RUN apk add --no-cache jq git curl bash openssl -RUN mkdir -p /code -COPY . /usr/src/app -WORKDIR /usr/src/app -RUN apk add --no-cache --virtual .build-deps gcc musl-dev libffi-dev openssl-dev -RUN pip install cython -RUN easy_install /usr/src/app -WORKDIR /code -ENTRYPOINT [ "/usr/src/app/run-scan.sh" ] diff --git a/Dockerfile.development b/Dockerfile.development deleted file mode 100644 index 0869ddcb..00000000 --- a/Dockerfile.development +++ /dev/null @@ -1,18 +0,0 @@ -FROM ubuntu:bionic -RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ - curl git python-tox \ - make build-essential libssl1.0-dev zlib1g-dev libbz2-dev \ - libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev \ - xz-utils tk-dev libffi-dev liblzma-dev -RUN curl -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer | bash -RUN mkdir -p /code -ENV PATH "/root/.pyenv/bin:$PATH" -#/usr/bin:/usr/sbin:/sbin:/bin:/usr/local/bin:/usr/local/sbin" -RUN echo 'export PATH="~/.pyenv/bin:$PATH"; eval "$(pyenv init -)"; eval "$(pyenv virtualenv-init -)"' >> ~/.bashrc -RUN pyenv install 2.7.15 -RUN pyenv install 3.5.6 -RUN pyenv install 3.6.6 -RUN pyenv install pypy-5.7.1 -RUN pyenv global 2.7.15 3.5.6 3.6.6 pypy-5.7.1 -WORKDIR /code -ENTRYPOINT bash -c 'eval "$(pyenv init -)"; eval "$(pyenv virtualenv-init -)"; make clean && make setup && make test' diff --git a/Dockerfiles/00.python.Dockerfile b/Dockerfiles/00.python.Dockerfile new file mode 100644 index 00000000..10d32e2e --- /dev/null +++ b/Dockerfiles/00.python.Dockerfile @@ -0,0 +1,7 @@ +FROM python:3 +LABEL maintainer="squad:git-defenders" url="https://github.ibm.com/whitewater/whitewater-detect-secrets" + +RUN apt-get -y remove --purge mysql* +# Remediate CVE-2019-18218 +RUN apt-get update && apt-get install file -y +RUN pip install --upgrade pip diff --git a/Dockerfiles/01.cli.Dockerfile b/Dockerfiles/01.cli.Dockerfile new file mode 100644 index 00000000..780572cb --- /dev/null +++ b/Dockerfiles/01.cli.Dockerfile @@ -0,0 +1,13 @@ +FROM git-defenders/python + +# Auto adjust line ending. Support running scan on Windows platform +RUN git config --global core.autocrlf true + +COPY README.md /code/ +COPY setup.py /code/ +COPY setup.cfg /code/ +COPY detect_secrets /code/detect_secrets + +RUN pip install /code + +WORKDIR /code diff --git a/Dockerfiles/01.dsl.Dockerfile b/Dockerfiles/01.dsl.Dockerfile new file mode 100644 index 00000000..3dca73a6 --- /dev/null +++ b/Dockerfiles/01.dsl.Dockerfile @@ -0,0 +1,9 @@ +FROM git-defenders/python + +RUN apt-get update && apt-get install -y jq +RUN mkdir -p /code +COPY . /usr/src/app +WORKDIR /usr/src/app +RUN pip install /usr/src/app +WORKDIR /code +ENTRYPOINT [ "/usr/src/app/run-scan.sh" ] diff --git a/Dockerfiles/02.detect-secrets-hook.Dockerfile b/Dockerfiles/02.detect-secrets-hook.Dockerfile new file mode 100644 index 00000000..729b11dc --- /dev/null +++ b/Dockerfiles/02.detect-secrets-hook.Dockerfile @@ -0,0 +1,4 @@ +FROM git-defenders/cli + +RUN git config --global core.safecrlf false +ENTRYPOINT [ "detect-secrets-hook" ] diff --git a/Dockerfiles/02.detect-secrets.Dockerfile b/Dockerfiles/02.detect-secrets.Dockerfile new file mode 100644 index 00000000..d2bfe004 --- /dev/null +++ b/Dockerfiles/02.detect-secrets.Dockerfile @@ -0,0 +1,3 @@ +FROM git-defenders/cli +ENTRYPOINT [ "detect-secrets" ] +CMD [ "scan", "/code" ] diff --git a/build-dockerfiles.sh b/build-dockerfiles.sh new file mode 100755 index 00000000..93b944b9 --- /dev/null +++ b/build-dockerfiles.sh @@ -0,0 +1,20 @@ +#!/bin/bash -ex + +CUR_DIR=$(dirname $0) +pushd "${CUR_DIR}" + +IMAGE_DOMAIN=git-defenders + +# build images +for dockerfile in Dockerfiles/*.Dockerfile +do + image_name=$(echo -e $(basename ${dockerfile}) | cut -d\. -f2) + docker build -f "${dockerfile}" -t $IMAGE_DOMAIN/$image_name . +done + +# test images +docker run -it --entrypoint detect-secrets $IMAGE_DOMAIN/dsl --version +docker run -it $IMAGE_DOMAIN/detect-secrets --version +docker run -it $IMAGE_DOMAIN/detect-secrets-hook --version + +popd