Rearchitect for v0.3.0 (#105)

* Stop to use postgreSQL * Remove SQLC workflow configuration * Fix gosec error * fix build issue * Update Dockerfile with Go 1.22 and Trivy 0.50.4 * Update Docker build and push actions, manage Go build cache * Add support for linux/arm64 in platforms configuration * Comment out platforms in publish workflow file * Add BigQuery client to infrastructure in New() function * remove debug code * Add firestore DB * Update go packages * Add firestore insertion * Add utility function to hash branch names * Add hex encoding to HashBranch and create hash_test.go * Use Cloud Storage for scan report repository instead of Firebase * Remove h1 * Save object data with gzip encoding * Update comment message * fix tests * test * ignore draft PR * Update ListIssueComments receiver to pointer type * Impelement GitHub check * Update check conclusion to "cancelled" in ScanGitHubRepo * Add impoersonation for BigQuery * Adjust impersonation scopes * Add policy client * Update GitHub comment format * Update references display in test and comment body template * Update comment_body.md template with consistent emoji usage * Update detected vulnerabilities section icon to warning sign * Add support for BigQuery service account impersonation
m-mizutani · Jun 9, 2024 · f1410dc · f1410dc
1 parent 85ca7d3
commit f1410dc
Show file tree

Hide file tree

Showing 89 changed files with 4,522 additions and 2,474 deletions.
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
@@ -11,31 +11,61 @@ env:
 jobs:
   build:
     runs-on: ubuntu-latest
-    if: github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/')
     steps:
       - name: checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+
+      - name: Go Build Cache for Docker
+        uses: actions/cache@v3
+        with:
+          path: go-build-cache
+          key: ${{ runner.os }}-go-build-cache-${{ hashFiles('go.sum') }}
+
+      - name: inject go-build-cache into docker
+        # v1 was composed of two actions: "inject" and "extract".
+        # v2 is unified to a single action.
+        uses: reproducible-containers/[email protected]
+        with:
+          cache-source: go-build-cache
+
       - name: Set up Docker buildx
-        uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226
+        uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0
       - name: Login to GitHub Container Registry
-        uses: docker/login-action@v1
+        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}
-      - name: Build and push Docker image
-        id: build-and-push
-        uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09
+
+      - name: Get the tag or commit id
+        id: version
+        run: |
+          if [[ $GITHUB_REF == refs/tags/* ]]; then
+            # If a tag is present, strip the 'refs/tags/' prefix
+            TAG_OR_COMMIT=$(echo $GITHUB_REF | sed 's/refs\/tags\///')
+            echo "This is a tag: $TAG_OR_COMMIT"
+          else
+            # If no tag is present, use the commit SHA
+            TAG_OR_COMMIT=$(echo $GITHUB_SHA)
+            echo "This is a commit SHA: $TAG_OR_COMMIT"
+          fi
+          # Set the variable for use in other steps
+          echo "TAG_OR_COMMIT=$TAG_OR_COMMIT" >> $GITHUB_OUTPUT
+        shell: bash
+
+      - name: Build and push
+        uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 # v5.1.0
         with:
           context: .
           push: true
           tags: ${{ env.GITHUB_IMAGE_NAME }}
+          build-args: |
+            BUILD_VERSION=${{ steps.version.outputs.TAG_OR_COMMIT }}
           cache-from: type=gha
           cache-to: type=gha,mode=max
-          platforms: linux/amd64,linux/arm64
-      - uses: actions/checkout@v2
+          # platforms: linux/amd64,linux/arm64
       - name: Slack Notification
-        uses: rtCamp/action-slack-notify@v2
+        uses: rtCamp/action-slack-notify@12e36fc18b0689399306c2e0b3e0f2978b7f1ee7 # v2.2.0
         env:
           SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
           SLACK_MESSAGE: "Pushed to ${{ env.GITHUB_IMAGE_NAME }}"

diff --git a/.github/workflows/sqlc.yml b/.github/workflows/sqlc.yml
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -24,12 +24,6 @@ jobs:
       - uses: actions/setup-go@v4
         with:
           go-version-file: "go.mod"
-      - run: |
-          curl -Lsf -O https://github.com/k0kubun/sqldef/releases/download/v0.16.5/psqldef_linux_amd64.tar.gz
-          tar xzf psqldef_linux_amd64.tar.gz
-          ./psqldef -U pguser -f ./database/schema.sql testdb
-        env:
-          PGPASSWORD: pgpass
       - run: go test --tags github ./...
         env:
           TEST_DB_DSN: "user=pguser password=pgpass dbname=testdb sslmode=disable"

diff --git a/.gitignore b/.gitignore
@@ -8,6 +8,7 @@ assets/.next
 /*.json
 
 tmp
+/pkg/usecase/templates/test_*
 
 trivy.db
 octovy

diff --git a/Dockerfile b/Dockerfile
@@ -1,16 +1,24 @@
-FROM golang:1.21-bullseye AS build-go
-COPY . /app
+FROM golang:1.22 AS build-go
+ENV CGO_ENABLED=0
+ARG BUILD_VERSION
+
 WORKDIR /app
-# ENV CGO_ENABLED=0
-RUN go get -v
-RUN go build .
+RUN go env -w GOMODCACHE=/root/.cache/go-build
+
+COPY go.mod go.sum ./
+RUN --mount=type=cache,target=/root/.cache/go-build go mod download
+
+COPY . /app
+RUN --mount=type=cache,target=/root/.cache/go-build go build -o octovy -ldflags "-X github.com/m-mizutani/octovy/pkg/domain/types.AppVersion=${BUILD_VERSION}" .
 
 FROM gcr.io/distroless/base:nonroot
+USER nonroot
 COPY --from=build-go /app/octovy /octovy
-COPY --from=build-go /app/database /database
-COPY --from=aquasec/trivy:0.45.1 /usr/local/bin/trivy /trivy
+COPY --from=aquasec/trivy:0.50.4 /usr/local/bin/trivy /trivy
 WORKDIR /
 ENV OCTOVY_ADDR="0.0.0.0:8000"
 ENV OCTOVY_TRIVY_PATH=/trivy
 EXPOSE 8000
-ENTRYPOINT [ "/octovy" ]
+
+ENTRYPOINT ["/octovy"]
+
diff --git a/README.md b/README.md
@@ -1,8 +1,15 @@
 # Octovy
 
-Octovy is a GitHub application designed to identify and alert you to any dependencies in your repository that could be potentially vulnerable. It uses [trivy](https://github.com/aquasecurity/trivy) for detection and then stores the results in a database for your reference.
+Octovy is a GitHub App that scans your repository's code for potentially vulnerable dependencies. It utilizes [trivy](https://github.com/aquasecurity/trivy) to detect software vulnerabilities. When triggered by events like `push` and `pull_request` from GitHub, Octovy scans the repository for dependency vulnerabilities and performs the following actions:
 
-![architecture](https://github.com/m-mizutani/octovy/assets/605953/a58c93e1-cfbf-4ff7-9427-1fc385cf7b9c)
+- Adds a comment to the pull request, summarizing the vulnerabilities found
+- Inserts the scan results into BigQuery
+
+![architecture](https://github.com/m-mizutani/octovy/assets/605953/4366161f-a4ff-4abb-9766-0fb4df818cb1)
+
+Octovy adds a comment to the pull request when it detects new vulnerabilities between the head of the PR and the merge destination.
+
+<img width="755" alt="comment example" src="https://github.com/m-mizutani/octovy/assets/605953/052a6362-c284-4857-921c-5c3c2f32065b">
 
 ## Setup
 
@@ -16,51 +23,46 @@ Start by creating a GitHub App [here](https://github.com/settings/apps). You can
 
 - **Permissions & events**
   - Repository Permissions
+    - **Checks**: Set to Read & Write
     - **Contents**: Set to Read-only
     - **Metadata**: Set to Read-only
     - **Pull Requests**: Set to Read & Write
   - Subscribe to events
     - **Pull request**
     - **Push**
 
-Once complete, note down the following information from the **General** section for later:
+Once you have completed the setup, make sure to take note of the following information from the **General** section for future reference:
 
 - **App ID** (e.g. `123456`)
 - **Private Key**: Click `Generate a private key` and download the key file (e.g. `your-app-name.2023-08-14.private-key.pem`)
 
-### 2. Setting Up the Database
-
-Octovy requires a PostgreSQL database. You can use any PostgreSQL instance you like, but we recommend cloud-based database services such as [Google Cloud SQL](https://cloud.google.com/sql) or [Amazon RDS](https://aws.amazon.com/rds/).
-
-For database migration, [sqldef](https://github.com/k0kubun/sqldef) is recommended. After installing sqldef, you can migrate your database schema using the command below. Be sure to replace the placeholders with your actual database information.
+### 2. Setting Up Cloud Resources
 
-```bash
-# NOTICE: Be careful not to save the password to shell history
-$ export PGPASSWORD=[db_password]
-$ psqldef -U [db_user] -p [db_port] -h [db_host] -f database/schema.sql [db_name]
-```
+- **Cloud Storage**: Create a Cloud Storage bucket dedicated to storing the scan results exclusively for Octovy's use.
+- **BigQuery** (Optional): Create a BigQuery dataset and table for storing the scan results. Octovy will automatically update the schema. The default table name should be `scans`.
 
 ### 3. Deploying Octovy
 
 The recommended method of deploying Octovy is via a container image, available at `ghcr.io/m-mizutani/octovy`. This image is built using GitHub Actions and published to the GitHub Container Registry.
 
 To run Octovy, set the following environment variables:
 
-- GitHub App
-  - `OCTOVY_GITHUB_APP_ID`: App ID of your GitHub App
-  - `OCTOVY_GITHUB_APP_PRIVATE_KEY`: Private key of your GitHub App
-  - `OCTOVY_GITHUB_SECRET`: Webhook secret of your GitHub App
-- Network
-  - `OCTOVY_ADDR`: Listening address (e.g. `0.0.0.0:8080`)
-- Database
-  - `OCTOVY_DB_HOST`: Hostname of your PostgreSQL database
-  - `OCTOVY_DB_PORT`: Port number of your PostgreSQL database
-  - `OCTOVY_DB_USER`: Username of your PostgreSQL database
-  - `OCTOVY_DB_PASSWORD`: Password of your PostgreSQL database
-  - `OCTOVY_DB_NAME`: Database name of your PostgreSQL database
-- Logging
-  - `OCTOVY_LOG_LEVEL`: Log level (e.g. `debug`, `info`, `warn`, `error`)
-  - `OCTOVY_LOG_FORMAT`: Log format, recommend to use `json`
+#### Required Environment Variables
+- `OCTOVY_ADDR`: The address to bind the server to (e.g. `:8080`)
+- `OCTOVY_GITHUB_APP_ID`: The GitHub App ID
+- `OCTOVY_GITHUB_APP_PRIVATE_KEY`: The path to the private key file
+- `OCTOVY_GITHUB_APP_SECRET`: The secret string used to verify the webhook request from GitHub
+- `OCTOVY_CLOUD_STORAGE_BUCKET`: The name of the Cloud Storage bucket
+
+#### Optional Environment Variables
+- `OCTOVY_TRIVY_PATH`: The path to the trivy binary. If you uses the our container image, you don't need to set this variable.
+- `OCTOVY_CLOUD_STORAGE_PREFIX`: The prefix for the Cloud Storage object
+- `OCTOVY_BIGQUERY_PROJECT_ID`: The name of the BigQuery dataset
+- `OCTOVY_BIGQUERY_DATASET_ID`: The name of the BigQuery table
+- `OCTOVY_BIGQUERY_TABLE_ID`: The name of the BigQuery table
+- `OCTOVY_BIGQUERY_IMPERSONATE_SERVICE_ACCOUNT`: The service account to impersonate when accessing BigQuery
+- `OCTOVY_SENTRY_DSN`: The DSN for Sentry
+- `OCTOVY_SENTRY_ENV`: The environment for Sentry
 
 ## License
 

diff --git a/database/embed.go b/database/embed.go
diff --git a/database/query.sql b/database/query.sql