Update Snapshot and Timestamp (#1062) #39
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # | |
| # Copyright 2021 The Sigstore Authors. | |
| # | |
| # Licensed under the Apache License, Version 2.0 (the "License"); | |
| # you may not use this file except in compliance with the License. | |
| # You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, software | |
| # distributed under the License is distributed on an "AS IS" BASIS, | |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| # See the License for the specific language governing permissions and | |
| # limitations under the License. | |
| # This sync will execute when any combination of the following files, | |
| # and no other files, are changed on the main branch: | |
| # - timestamp.json | |
| # - snapshot.json | |
| # - [0-9]+.snapshot.json | |
| # Under this condition, all files from the repository/repository directory | |
| # on the main branch will sync to both preprod and prod. | |
| name: Sync Repository Main Branch with both GCS Preprod and Prod Buckets | |
| on: | |
| push: | |
| branches: | |
| - main | |
| paths: | |
| # When timestamp or snapshot files are changed. | |
| # Note: the sync job below uses a diff to ensure ONLY these files are changed | |
| # prior to syncing. | |
| - 'repository/repository/timestamp.json' | |
| - 'repository/repository/snapshot.json' | |
| - 'repository/repository/[0-9]+.snapshot.json' | |
| workflow_dispatch: | |
| jobs: | |
| sync: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| id-token: 'write' | |
| steps: | |
| - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 | |
| with: | |
| fetch-depth: 0 | |
| - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0 | |
| with: | |
| go-version-file: './go.mod' | |
| check-latest: true | |
| # Setup OIDC->SA auth | |
| - uses: google-github-actions/auth@67e9c72af6e0492df856527b474995862b7b6591 # v2.0.0 | |
| id: auth | |
| with: | |
| token_format: 'access_token' | |
| workload_identity_provider: 'projects/237800849078/locations/global/workloadIdentityPools/root-signing-pool/providers/sigstore-root' | |
| service_account: '[email protected]' | |
| create_credentials_file: true | |
| - uses: google-github-actions/setup-gcloud@5a5f7b85fca43e76e53463acaa9d408a03c98d3a # v2.0.1 | |
| with: | |
| project_id: project-rekor | |
| - name: Login | |
| run: | | |
| gcloud auth login --brief --cred-file="${{ steps.auth.outputs.credentials_file_path }}" | |
| gcloud auth list | |
| # Sync | |
| - name: sync | |
| run: | | |
| check_expiration() { | |
| expiry=$(jq -r '.signed.expires' $1) | |
| expires=$(date -d $expiry +%s) | |
| current=$(date +%s) | |
| if (( expires < current )); then | |
| echo "Detected expired metadata file $1 at $expiry!" | |
| exit 1 | |
| fi; | |
| } | |
| # Checks whether a filename matches timestamp.json, snapshot.json, or [0-9]+.snapshot.json. If not, | |
| # this workflow will exit as we only want to run it when ONLY these files are changed. | |
| # TODO it may be good to check whether the [0-9]+.snapshot.json is the next one chronologically | |
| check_filename() { | |
| if [[ $1 != "timestamp.json" && $1 != "snapshot.json" && !($1 =~ ^[0-9]+\.snapshot.json$) ]]; then | |
| echo "Sync main to preprod and prod workflow: Files other than timestamp and snapshot were updated in main branch, including file: $1. Not syncing, exiting." | |
| exit 0 | |
| fi; | |
| } | |
| # Download bucket metadata | |
| gcloud --quiet storage cp -r gs://sigstore-tuf-root/ . | |
| # Diff main and prod to determine whether ONLY the timestamp and snapshot files have changed in main. | |
| # If other files have also changed, exit - in this case, the sync should be to preprod only. | |
| # NOTE other non-timestamp/snapshot changes should only occur during a ceremony, and | |
| # will go through the sync-ceremony-* flow that hits main and preprod. This means there should never | |
| # be changed files in prod that have not also hit the main branch. | |
| # NOTE We deliberately diff only with prod to avoid a scenario in which the ceremony branch's sync to | |
| # main and preprod kicks off this workflow, in which case the lack of diff between main and preprod | |
| # could trigger this workflow to auto sync to prod. | |
| # | |
| # TODO this does not check whether the updates are in main or in prod, only that files differ. We could | |
| # make this more exact later to check that the updates are in main (anything else is unexpected). | |
| diff -qr repository/repository sigstore-tuf-root | grep -Po '([0-9\.]*\w+[\.\w+]*(?= differ))|((Only in \w+\: )\K(.*))' | while read l; do check_filename $l; done | |
| # Upload all but TUF timestamp. Once timestamp is uploaded, all other files must have been uploaded. | |
| for f in $(ls repository/repository/ -I *timestamp.json) | |
| do | |
| # Check for expiration if this is a non-versioned metadata file. | |
| # Versioned metadata like 1.root.json may be expired. | |
| # TODO(asraa): When consistent snapshots are enabled, this logic must be changed so that | |
| # only old versioned metadata can be expired. | |
| if [[ $f == [^0-9]*.json ]]; then | |
| check_expiration repository/repository/$f | |
| fi; | |
| gcloud --quiet storage cp --cache-control=no-store -r repository/repository/$f gs://sigstore-preprod-tuf-root/ | |
| gcloud --quiet storage cp --cache-control=no-store -r repository/repository/$f gs://sigstore-tuf-root/ | |
| done | |
| # Upload timestamp after checking latest timestamp expiration | |
| check_expiration repository/repository/timestamp.json | |
| gcloud --quiet storage cp --cache-control=no-store -r repository/repository/*timestamp.json gs://sigstore-preprod-tuf-root/ | |
| gcloud --quiet storage cp --cache-control=no-store -r repository/repository/*timestamp.json gs://sigstore-tuf-root/ | |
| # NOTE as this workflow runs only when timestamp or snapshot files are added or updated, there should not | |
| # be a scenario where files that are removed from main must be synced to (removed from) preprod/prod. | |
| gcloud compute url-maps invalidate-cdn-cache tuf-preprod-repo-cdn-lb --path "/*" --async | |
| gcloud compute url-maps invalidate-cdn-cache tuf-repo-cdn-lb --path "/*" --async | |
| if-failed: | |
| runs-on: ubuntu-latest | |
| needs: [sync] | |
| permissions: | |
| issues: 'write' | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| ISSUE_REPOSITORY: sigstore/root-signing | |
| if: always() && needs.sync.result == 'failure' | |
| steps: | |
| - name: Create issue on failure | |
| uses: sigstore/sigstore-probers/.github/actions/create-issue@main | |
| with: | |
| issue_repository: sigstore/root-signing | |
| issue_type: FAILURE |