go mod download in go-test-oss

hashicorp · Jul 21, 2023 · d6c1144 · d6c1144
1 parent 7e6ce76
commit d6c1144
Show file tree

Hide file tree

Showing 8 changed files with 218 additions and 2 deletions.
diff --git a/.github/scripts/cachekey_GOCACHE.sh b/.github/scripts/cachekey_GOCACHE.sh
@@ -0,0 +1,5 @@
+#!/bin/bash
+
+set -euo pipefail
+
+echo "{cachekey_GOCACHE}={GOCACHEv0:${GOVERSION}}:${GOMODULENAME}:${GOARCH}:${GOTAGS}" >> "$GITHUB_OUTPUT"
diff --git a/.github/scripts/cachekey_GOMODCACHE.sh b/.github/scripts/cachekey_GOMODCACHE.sh
@@ -0,0 +1,6 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# TODO: do we care about go version? I don't think so?
+echo "{cachekey_GOMODCACHE}={GOMODCACHEv0:${GOMODULENAME}}" >> "$GITHUB_OUTPUT"
diff --git a/.github/workflows/go-tests.yml b/.github/workflows/go-tests.yml
@@ -112,11 +112,13 @@ jobs:
     - uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1
       with:
         go-version-file: 'go.mod'
+        cache: false
     - run: go install github.com/reillywatson/enumcover/cmd/enumcover@master && enumcover ./...
     - name: Notify Slack
       if: ${{ failure() }}
       run: .github/scripts/notify_slack.sh
 
+
   lint-container-test-deps:
     needs:
     - setup
@@ -173,14 +175,44 @@ jobs:
     secrets:
       elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
 
+  # TODO: matrix?
+  go-cache-key-GOMODCACHE_root:
+    needs:
+    - setup
+    runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }}
+    env:
+      GOMODULENAME: "github.com/hashicorp/consul"
+    steps:
+      - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+      - id: main
+        run: |
+          .github/scripts/cachekey_GOMODCACHE.sh
+    outputs:
+      key: "${{ steps.main.outputs.cachekey_GOMODCACHE }}"
+
+  # TODO: matrix?
+  go-mod-download_root:
+    needs:
+    - setup
+    - go-cache-key-GOMODCACHE_root
+    uses: ./.github/workflows/reusable-go-mod-download.yml
+    with:
+      directory: "."
+      cachekey_GOMODCACHE: "${{ needs.go-cache-key-GOMODCACHE_root.outputs.key }}"
+      repository-name: ${{ github.repository }}
+      # XL is probably overkill?
+      runs-on: ${{ needs.setup.outputs.compute-xl }}
+
   # create a development build
   dev-build:
     needs:
     - setup
+    - go-cache-key-GOMODCACHE_root
     uses: ./.github/workflows/reusable-dev-build.yml
     with:
       runs-on: ${{ needs.setup.outputs.compute-xl }}
       repository-name: ${{ github.repository }}
+      cachekey_GOMODCACHE: "${{ needs.go-cache-key-GOMODCACHE_root.outputs.key }}"
     secrets:
       elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
 
@@ -194,6 +226,7 @@ jobs:
       runs-on: ${{ needs.setup.outputs.compute-xl }}
       go-arch: "s390x"
       repository-name: ${{ github.repository }}
+      # TODO: cachekey
     secrets:
       elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
 
@@ -233,6 +266,7 @@ jobs:
   go-test-oss:
     needs:
     - setup
+    - go-cache-key-GOMODCACHE_root
     - dev-build
     uses: ./.github/workflows/reusable-unit-split.yml
     with:
@@ -241,6 +275,7 @@ jobs:
       runs-on: ${{ needs.setup.outputs.compute-xl }}
       repository-name: ${{ github.repository }}
       go-tags: ""
+      cachekey_GOMODCACHE: "${{ needs.go-cache-key-GOMODCACHE_root.outputs.key }}"
     permissions:
       id-token: write # NOTE: this permission is explicitly required for Vault auth.
       contents: read

diff --git a/.github/workflows/reusable-check-go-mod.yml b/.github/workflows/reusable-check-go-mod.yml
@@ -13,6 +13,9 @@ on:
     secrets:
       elevated-github-token:
         required: true
+    outputs:
+      go-version:
+        value: "${{ jobs.check-go-mod.outputs.go-version }}"
 jobs:
   check-go-mod:
     runs-on: ${{ fromJSON(inputs.runs-on) }}
@@ -23,7 +26,8 @@ jobs:
     - name: Setup Git
       if: ${{ endsWith(inputs.repository-name, '-enterprise') }}
       run: git config --global url."https://${{ secrets.elevated-github-token }}:@github.com".insteadOf "https://github.com"
-    - uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1
+    - id: setup_go 
+      uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1
       with:
         go-version-file: 'go.mod'
     - run: go mod tidy
@@ -36,3 +40,6 @@ jobs:
     - name: Notify Slack
       if: ${{ failure() }}
       run: .github/scripts/notify_slack.sh
+
+    outputs:
+      go-version: "${{ steps.setup_go.outputs.go-version }}"
diff --git a/.github/workflows/reusable-dev-build.yml b/.github/workflows/reusable-dev-build.yml
@@ -18,6 +18,10 @@ on:
         required: false
         type: string
         default: ""
+      cachekey_GOMODCACHE:
+        required: false
+        type: string
+        default: ""
     secrets:
       elevated-github-token:
         required: true
@@ -33,6 +37,15 @@ jobs:
       - uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1
         with:
           go-version-file: 'go.mod'
+          cache: false
+      - name: restore GOMODCACHE
+        uses: actions/cache/restore@v3
+        if: "${{ inputs.cachekey_GOMODCACHE != '' }}"
+        with:
+          key: "${{inputs.cachekey_GOMODCACHE}}"
+          path: ~/go/pkg/mod 
+      # this should take no time at all, since we should hit GOMODCACHE
+      - run: go mod download
       - name: Build
         env:
           GOARCH: ${{ inputs.goarch }}

diff --git a/.github/workflows/reusable-go-mod-download.yml b/.github/workflows/reusable-go-mod-download.yml
@@ -0,0 +1,49 @@
+name: reusable-go-mod-download
+
+on:
+  workflow_call:
+    inputs:
+      directory:
+        required: true
+        type: string
+      runs-on:
+        description: An expression indicating which kind of runners to use.
+        required: true
+        type: string
+      repository-name:
+        required: true
+        type: string
+      cachekey_GOMODCACHE:
+        required: true
+        type: string
+
+env:
+  GOPRIVATE: github.com/hashicorp # Required for enterprise deps
+
+jobs:
+  go-test:
+    runs-on: ${{ fromJSON(inputs.runs-on) }}
+    steps:      
+      - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2
+      # NOTE: This step is specifically needed for ENT. It allows us to access the required private HashiCorp repos.
+      - name: Setup Git
+        if: ${{ endsWith(inputs.repository-name, '-enterprise') }}
+        run: git config --global url."https://${{ secrets.elevated-github-token }}:@github.com".insteadOf "https://github.com"
+
+      - id: setup_go
+        uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1
+        with:
+          go-version-file: '${{inputs.directory}}/go.mod'
+          cache: false
+      - name: restore GOMODCACHE
+        uses: actions/cache/restore@v3
+        with:
+          key: "${{inputs.cachekey_GOMODCACHE}}"
+          path: ~/go/pkg/mod
+      - run: go mod download
+        working-directory: ${{inputs.directory}}
+      - name: save GOMODCACHE
+        uses: actions/cache/restore@v3
+        with:
+          key: "${{inputs.cachekey_GOMODCACHE}}"
+          path: ~/go/pkg/mod
diff --git a/.github/workflows/reusable-go-test-build.yml b/.github/workflows/reusable-go-test-build.yml
@@ -0,0 +1,88 @@
+name: reusable-go-test-build 
+
+on:
+  workflow_call:
+    inputs:
+      directory:
+        required: true
+        type: string
+      runs-on:
+        description: An expression indicating which kind of runners to use.
+        required: true
+        type: string
+      cachekey_GOMODCACHE:
+        required: false
+        type: string
+      package-names-command:
+        required: false
+        type: string
+        default: 'go list -tags "$GOTAGS" ./...'
+      go-arch:
+        required: false
+        type: string
+        default: ""
+      go-test-flags:
+        required: false
+        type: string
+        default: ""
+      go-tags:
+        required: false
+        type: string
+        default: ""
+      repository-name:
+        required: true
+        type: string
+    secrets:
+      elevated-github-token:
+        required: true
+      consul-license:
+        required: true
+      datadog-api-key:
+        required: true
+env:
+  # TODO: the default should probably be derived from the runner
+  GOARCH: ${{ inputs.go-arch == '' && 'amd64' || inputs.go-arch }}
+  GOTAGS: ${{ inputs.go-tags}}
+  GOPRIVATE: github.com/hashicorp # Required for enterprise deps
+
+jobs:
+  go-test:
+    runs-on: ${{ fromJSON(inputs.runs-on) }}
+    steps:      
+      - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2
+      # NOTE: This step is specifically needed for ENT. It allows us to access the required private HashiCorp repos.
+      - name: Setup Git
+        if: ${{ endsWith(inputs.repository-name, '-enterprise') }}
+        run: git config --global url."https://${{ secrets.elevated-github-token }}:@github.com".insteadOf "https://github.com"
+      - id: setup_go
+        uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1
+        with:
+          go-version-file: '${{inputs.directory}}/go.mod'
+          cache: false
+      - id: vars
+        working-directory: ${{inputs.directory}}
+        run: |
+          echo "{key_GOMODCACHE}={GOMODCACHEv0:$(go list -m)}" >> "$GITHUB_OUTPUT"
+          # TODO: does GOTAGS need escaping? normalizing? sorting?
+          # TODO: other flags?
+          echo "{key_GOCACHE}={GOCACHEv0:$(go list -m):${{ steps.setup_go.outputs.go-version }}:${[ env.GOARCH }}:${{ env.GOTAGS }}" >> "$GITHUB_OUTPUT"
+      - name: restore GOMODCACHE
+        uses: actions/cache/restore@v3
+        with:
+          key: "${{steps.vars.outputs.key_GOMODCACHE}}"
+          path: ~/go/pkg/mod
+      - name: restore GOCACHE
+        uses: actions/cache/restore@v3
+        with:
+          key: "${{steps.vars.outputs.key_GOCACHE}}"
+          path: ~/.cache/go-build
+      - name: Build tests 
+        working-directory: ${{inputs.directory}}
+        run: |
+            PACKAGE_NAMES=$(${{inputs.package-names-command}})
+            go test -tags "${{env.GOTAGS}}" -c ${PACKAGE_NAMES}
+      - name: save GOCACHE
+        uses: actions/cache/save@v3
+        with:
+          key: "${{steps.vars.outputs.key_GOCACHE}}"
+          path: ~/.cache/go-build
diff --git a/.github/workflows/reusable-unit-split.yml b/.github/workflows/reusable-unit-split.yml
@@ -37,6 +37,10 @@ on:
         required: false
         type: string
         default: ""
+      cachekey_GOMODCACHE:
+        required: false
+        type: string
+        default: ""
     secrets:
       elevated-github-token:
         required: true
@@ -61,9 +65,11 @@ jobs:
       package-matrix: ${{ steps.set-matrix.outputs.matrix }}
     steps:
       - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+      # TODO: not too sure why we do this setup for every package?
       - uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1
         with:
           go-version-file: 'go.mod'
+          cache: false
       - id: set-matrix
         run: ./.github/scripts/set_test_package_matrix.sh ${{env.TOTAL_RUNNERS}}
 
@@ -91,8 +97,15 @@ jobs:
       - uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1
         with:
           go-version-file: 'go.mod'
-          cache: true
+          cache: false
+      - name: restore GOMODCACHE
+        uses: actions/cache/restore@v3
+        if: "${{ inputs.cachekey_GOMODCACHE != '' }}"
+        with:
+          key: "${{inputs.cachekey_GOMODCACHE}}"
+          path: ~/go/pkg/mod      
       - run: mkdir -p ${{env.TEST_RESULTS}}
+      # expect this to be instant if cache hit
       - name: go mod download
         working-directory: ${{inputs.directory}}
         run: go mod download