diff --git a/.github/linters/prometheus-prefect-exporter-ct.yaml b/.github/linters/prometheus-prefect-exporter-ct.yaml index c54368e1..88847912 100644 --- a/.github/linters/prometheus-prefect-exporter-ct.yaml +++ b/.github/linters/prometheus-prefect-exporter-ct.yaml @@ -4,5 +4,5 @@ charts: chart-repos: - bitnami=https://charts.bitnami.com/bitnami helm-extra-args: --timeout 600s -namespace: default +namespace: prefect release-label: prefect diff --git a/.github/linters/server-ct.yaml b/.github/linters/server-ct.yaml index f33fe015..54cc9211 100644 --- a/.github/linters/server-ct.yaml +++ b/.github/linters/server-ct.yaml @@ -4,3 +4,5 @@ charts: chart-repos: - bitnami=https://charts.bitnami.com/bitnami helm-extra-args: --timeout 90s +namespace: prefect +release-label: prefect diff --git a/.github/workflows/deploy-latest-helm-version.yaml b/.github/workflows/deploy-latest-helm-version.yaml index fcb309f0..d269c07b 100644 --- a/.github/workflows/deploy-latest-helm-version.yaml +++ b/.github/workflows/deploy-latest-helm-version.yaml @@ -1,10 +1,14 @@ -name: Deploy latest helm releases to internal cluster +--- +name: Deploy Latest Helm Releases to Internal Cluster "on": workflow_dispatch: inputs: release_version: - description: Release tag + description: The latest release tag + required: true + +permissions: {} jobs: update_helm_releases: @@ -21,4 +25,4 @@ jobs: --ref main \ -f release_version=${{ github.event.inputs.release_version }} env: - GITHUB_TOKEN: ${{ secrets.OPS_CLUSTER_DEPLOYMENT_ACTIONS_RW }} + GH_TOKEN: ${{ secrets.OPS_CLUSTER_DEPLOYMENT_ACTIONS_RW }} diff --git a/.github/workflows/helm-release.yaml b/.github/workflows/helm-release.yaml index 8ec6d7c6..b4b41b3c 100644 --- a/.github/workflows/helm-release.yaml +++ b/.github/workflows/helm-release.yaml @@ -1,11 +1,17 @@ -name: Release Helm Chart +--- +name: Release Prefect Server and Worker Helm Charts "on": - workflow_dispatch: + workflow_dispatch: {} + +permissions: {} jobs: release: runs-on: ubuntu-latest + permissions: + # GitHub considers creating releases and uploading assets as writing contents. + contents: write steps: - name: Checkout uses: actions/checkout@v4 diff --git a/.github/workflows/helm-unittest.yaml b/.github/workflows/helm-unittest.yaml index 922830c5..933aa988 100644 --- a/.github/workflows/helm-unittest.yaml +++ b/.github/workflows/helm-unittest.yaml @@ -1,10 +1,9 @@ -# https://github.com/marketplace/actions/helm-unit-tests +--- +name: Run Helm Unit Tests -name: Run Helm unit tests +"on": + pull_request: {} -on: pull_request - -# Do not grant jobs any permissions by default permissions: {} jobs: @@ -15,6 +14,7 @@ jobs: contents: read steps: - uses: actions/checkout@v4 + - uses: d3adb5/helm-unittest-action@v2 with: helm-version: v3.15.3 diff --git a/.github/workflows/notify-on-failure.yaml b/.github/workflows/notify-on-failure.yaml new file mode 100644 index 00000000..87d63857 --- /dev/null +++ b/.github/workflows/notify-on-failure.yaml @@ -0,0 +1,49 @@ +--- +name: Notify on Failure + +"on": + workflow_run: + workflows: + - Deploy Latest Helm Releases to Internal Cluster + - Release Prefect Server and Worker Helm Charts + - Release Prometheus Prefect Exporter Helm Chart + - Updatecli Major Dependency Updates + - Updatecli Minor Dependency Updates + types: [completed] + +permissions: {} + +jobs: + notify: + name: Notify on Failure + runs-on: ubuntu-latest + if: ${{ github.event.workflow_run.conclusion == 'failure' }} + permissions: + # required to introspect the workflow run + actions: read + # required to read from the repo + contents: read + steps: + - name: Format date + run: | + formatted_date=$(date -d "${{ github.event.workflow_run.run_started_at }}" "+%b %d at %I:%M %p") + echo "FORMATTED_DATE=$formatted_date" >> $GITHUB_ENV + + - name: Send Slack notification + uses: 8398a7/action-slack@v3 + with: + status: custom + # https://api.slack.com/reference/messaging/attachments + custom_payload: | + { + attachments: [{ + pretext: ':x: Workflow triggered by ${{ github.actor }} failed', + title: '${{ github.event.workflow_run.display_title }} #${{github.event.workflow_run.run_number}}', + title_link: '${{ github.event.workflow_run.html_url }}', + footer: '${{ github.repository }} | ${{ env.FORMATTED_DATE }} UTC', + footer_icon: 'https://slack-imgs.com/?c=1&o1=wi32.he32.si&url=https%3A%2F%2Fslack.github.com%2Fstatic%2Fimg%2Ffavicon-neutral.png', + color: 'danger', + }] + } + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_GHA_JOB_STATUS }} diff --git a/.github/workflows/prometheus-exporter-helm-release.yaml b/.github/workflows/prometheus-exporter-helm-release.yaml index 442a3eed..42507b59 100644 --- a/.github/workflows/prometheus-exporter-helm-release.yaml +++ b/.github/workflows/prometheus-exporter-helm-release.yaml @@ -1,11 +1,17 @@ +--- name: Release Prometheus Prefect Exporter Helm Chart "on": - workflow_dispatch: + workflow_dispatch: {} + +permissions: {} jobs: release: runs-on: ubuntu-latest + permissions: + # GitHub considers creating releases and uploading assets as writing contents. + contents: write steps: - name: Checkout uses: actions/checkout@v4 diff --git a/.github/workflows/prometheus-prefect-exporter-lint-and-test.yaml b/.github/workflows/prometheus-prefect-exporter-lint-and-test.yaml index d99fdf92..91dc3698 100644 --- a/.github/workflows/prometheus-prefect-exporter-lint-and-test.yaml +++ b/.github/workflows/prometheus-prefect-exporter-lint-and-test.yaml @@ -1,3 +1,4 @@ +--- name: Lint and Test Prometheus Prefect Exporter Chart "on": @@ -5,10 +6,15 @@ name: Lint and Test Prometheus Prefect Exporter Chart branches: - main +permissions: {} + jobs: lint_test: name: "lint-test (${{ matrix.kubernetes }})" runs-on: ubuntu-latest + permissions: + # required to read from the repo + contents: read strategy: matrix: kubernetes: diff --git a/.github/workflows/server-lint-and-test.yaml b/.github/workflows/server-lint-and-test.yaml index 6e52a22d..46c1e2f9 100644 --- a/.github/workflows/server-lint-and-test.yaml +++ b/.github/workflows/server-lint-and-test.yaml @@ -1,3 +1,4 @@ +--- name: Lint and Test Prefect Server Chart "on": @@ -5,7 +6,6 @@ name: Lint and Test Prefect Server Chart branches: - main -# Do not grant jobs any permissions by default permissions: {} jobs: diff --git a/.github/workflows/update-helm-major-versions.yaml b/.github/workflows/update-helm-major-versions.yaml deleted file mode 100644 index 3af328cd..00000000 --- a/.github/workflows/update-helm-major-versions.yaml +++ /dev/null @@ -1,45 +0,0 @@ ---- -name: Updatecli major dependency updates - -on: - workflow_dispatch: - schedule: - # ┌───────────── minute (0 - 59) - # │ ┌───────────── hour (0 - 23) - # │ │ ┌───────────── day of the month (1 - 31) - # │ │ │ ┌───────────── month (1 - 12 or JAN-DEC) - # │ │ │ │ ┌───────────── day of the week (0 - 6 or SUN-SAT) - # │ │ │ │ │ - # │ │ │ │ │ - # │ │ │ │ │ - - cron: 0 15 1 * * # The first of each month at 10am EST - -permissions: {} - -jobs: - updatecli: - runs-on: ubuntu-latest - permissions: - # required to write to the repo - contents: write - steps: - - name: checkout - uses: actions/checkout@v4 - - - name: configure git - run: | - git config user.name "$GITHUB_ACTOR" - git config user.email "$GITHUB_ACTOR@users.noreply.github.com" - - - name: Get current date - id: date - run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT - - - name: install updatecli in the runner - uses: updatecli/updatecli-action@v2 - - - name: run updatecli in diff mode - run: | - updatecli diff --config .github/updatecli/manifest-major.yaml - env: - GITHUB_TOKEN: ${{ github.token }} diff --git a/.github/workflows/update-helm-minor-versions.yaml b/.github/workflows/update-helm-minor-versions.yaml deleted file mode 100644 index 2542550f..00000000 --- a/.github/workflows/update-helm-minor-versions.yaml +++ /dev/null @@ -1,60 +0,0 @@ ---- -name: Updatecli minor dependency updates - -on: - workflow_dispatch: - schedule: - # ┌───────────── minute (0 - 59) - # │ ┌───────────── hour (0 - 23) - # │ │ ┌───────────── day of the month (1 - 31) - # │ │ │ ┌───────────── month (1 - 12 or JAN-DEC) - # │ │ │ │ ┌───────────── day of the week (0 - 6 or SUN-SAT) - # │ │ │ │ │ - # │ │ │ │ │ - # │ │ │ │ │ - - cron: 0 15 1 * * # The first of each month at 10am EST - -permissions: {} - -jobs: - updatecli: - runs-on: ubuntu-latest - permissions: - # required to write to the repo - contents: write - # required to open a pr with updatecli changes - pull-requests: write - steps: - - name: checkout - uses: actions/checkout@v4 - - - name: configure git - run: | - git config user.name "$GITHUB_ACTOR" - git config user.email "$GITHUB_ACTOR@users.noreply.github.com" - - - name: Get current date - id: date - run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT - - - name: create branch for helm version updates - run: | - git checkout -b "helm-version-${{ steps.date.outputs.date }}" - - - name: install updatecli in the runner - uses: updatecli/updatecli-action@v2 - - - name: run updatecli in apply mode - run: | - updatecli apply --config .github/updatecli/manifest-minor.yaml - git commit -am "helm-version-${{ steps.date.outputs.date }}" - git push --set-upstream origin "helm-version-${{ steps.date.outputs.date }}" - env: - GITHUB_TOKEN: ${{ github.token }} - - - name: create pr - run: | - git checkout "helm-version-${{ steps.date.outputs.date }}" - gh pr create --base main --title "helm-version-bump-${{ steps.date.outputs.date }}" -b "please run helm-docs locally to update chart readmes" --label dependencies - env: - GITHUB_TOKEN: ${{ github.token }} diff --git a/.github/workflows/updatecli-major-versions.yaml b/.github/workflows/updatecli-major-versions.yaml new file mode 100644 index 00000000..ec0aba8e --- /dev/null +++ b/.github/workflows/updatecli-major-versions.yaml @@ -0,0 +1,27 @@ +--- +name: Updatecli Major Dependency Updates + +"on": + schedule: + - cron: 0 15 1 * * # First of the month @ 3pm UTC + workflow_dispatch: {} + +permissions: {} + +jobs: + updatecli_major: + runs-on: ubuntu-latest + permissions: + # required to write to the repo + contents: write + # required to open a pr with updatecli changes + pull-requests: write + steps: + - name: checkout + uses: actions/checkout@v4 + + - name: updatecli-major-apply + uses: prefecthq/actions-updatecli-apply@main + with: + manifest-path: .github/updatecli/manifest-major.yaml + run-type: major diff --git a/.github/workflows/updatecli-minor-versions.yaml b/.github/workflows/updatecli-minor-versions.yaml new file mode 100644 index 00000000..e532f7d8 --- /dev/null +++ b/.github/workflows/updatecli-minor-versions.yaml @@ -0,0 +1,27 @@ +--- +name: Updatecli Minor Dependency Updates + +"on": + schedule: + - cron: 0 15 * * 1 # Monday @ 3pm UTC + workflow_dispatch: {} + +permissions: {} + +jobs: + updatecli_minor: + runs-on: ubuntu-latest + permissions: + # required to write to the repo + contents: write + # required to open a pr with updatecli changes + pull-requests: write + steps: + - name: checkout + uses: actions/checkout@v4 + + - name: updatecli-minor-apply + uses: prefecthq/actions-updatecli-apply@main + with: + manifest-path: .github/updatecli/manifest-minor.yaml + run-type: minor diff --git a/.github/workflows/validate-updatecli-config.yaml b/.github/workflows/validate-updatecli-config.yaml index 55e17c8e..e4986b43 100644 --- a/.github/workflows/validate-updatecli-config.yaml +++ b/.github/workflows/validate-updatecli-config.yaml @@ -1,23 +1,21 @@ --- -name: Validate updatecli config +name: Validate Updatecli Config -on: +"on": pull_request: branches: - main paths: - .github/updatecli/** -# Do not grant jobs any permissions by default permissions: {} jobs: - validate_updatecli_config: + validate_updatecli_configs: runs-on: ubuntu-latest permissions: # required to read from the repo contents: read - steps: - name: checkout uses: actions/checkout@v4 @@ -25,9 +23,8 @@ jobs: - name: install updatecli uses: updatecli/updatecli-action@v2 - - name: run updatecli diff to validate config - run: | - updatecli diff --config .github/updatecli/manifest-minor.yaml - updatecli diff --config .github/updatecli/manifest-major.yaml - env: - GITHUB_TOKEN: ${{ github.token }} + - name: validate manifest-minor + run: updatecli diff --config .github/updatecli/manifest-minor.yaml + + - name: validate manifest-major + run: updatecli diff --config .github/updatecli/manifest-major.yaml diff --git a/.github/workflows/worker-lint-and-test.yaml b/.github/workflows/worker-lint-and-test.yaml index 5197414f..5a337d1e 100644 --- a/.github/workflows/worker-lint-and-test.yaml +++ b/.github/workflows/worker-lint-and-test.yaml @@ -1,3 +1,4 @@ +--- name: Lint and Test Prefect Worker Chart "on": @@ -5,7 +6,6 @@ name: Lint and Test Prefect Worker Chart branches: - main -# Do not grant jobs any permissions by default permissions: {} jobs: diff --git a/.mise.toml b/.mise.toml index 141de490..4864c64d 100644 --- a/.mise.toml +++ b/.mise.toml @@ -1,5 +1,9 @@ [tools] -helm-docs = '1.13.1' -pre-commit = '3.7.1' +actionlint = '1.7.1' +github-cli = '2.52.0' helm = '3.15' helm-ct = '3.11.0' +helm-docs = '1.13.1' +pre-commit = '3.8.0' +shellcheck = '0.10.0' +yamllint = '1.35.1' diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 45576f61..3dd775de 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,19 +3,39 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v3.1.0 hooks: + - id: check-merge-conflict + - id: detect-private-key - id: no-commit-to-branch - id: trailing-whitespace + - repo: https://github.com/norwoodj/helm-docs rev: v1.11.0 hooks: - id: helm-docs args: - --template-files=README.md.gotmpl + - repo: https://github.com/gruntwork-io/pre-commit rev: v0.1.17 hooks: - id: helmlint + - repo: https://github.com/rhysd/actionlint rev: v1.7.1 hooks: - id: actionlint + args: + - -shellcheck= + + - repo: https://github.com/koalaman/shellcheck-precommit + rev: v0.7.2 + hooks: + - id: shellcheck + args: ["--severity=error"] + + - repo: https://github.com/adrienverge/yamllint.git + rev: v1.28.0 + hooks: + - id: yamllint + args: + - --strict \ No newline at end of file diff --git a/.yamllint b/.yamllint new file mode 100644 index 00000000..341b170c --- /dev/null +++ b/.yamllint @@ -0,0 +1,12 @@ +--- +extends: default + +ignore: | + charts/ + +rules: + comments: + min-spaces-from-content: 1 + comments-indentation: disable + document-start: disable + line-length: disable diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..f64783ed --- /dev/null +++ b/Makefile @@ -0,0 +1,57 @@ +# Worker Variables +WORKER_CHART_NAME=prefect-worker +WORKER_CHART_PATH=./charts/prefect-worker +WORKER_RELEASE_NAME=prefect-worker +NAMESPACE=prefect +VALUES_FILE=./charts/prefect-worker/values.yaml +# Server Variables +SERVER_CHART_NAME=prefect-server +SERVER_CHART_PATH=./charts/prefect-server +SERVER_RELEASE_NAME=prefect-server +NAMESPACE=prefect +VALUES_FILE=./charts/prefect-server/values.yaml +# Prometheus Prefect Exporter Variables +PROMETHEUS_PREFECT_EXPORTER_CHART_NAME=prometheus-prefect-exporter +PROMETHEUS_PREFECT_EXPORTER_CHART_PATH=./charts/prometheus-prefect-exporter +WORKER_RELEASE_NAME=prefect-worker +NAMESPACE=prefect +VALUES_FILE=./charts/prometheus-prefect-exporter/values.yaml + +.PHONY: all +all: tools + +.PHONY: mise +mise: + @mise install --yes + +.git/hooks/pre-commit: + @pre-commit install + +.PHONY: tools +tools: mise .git/hooks/pre-commit + +.PHONY: tools-list +tools-list: + @mise list --current + +.PHONY: buildprom +buildprom: ## Build Prometheus Prefect Exporter Helm dependencies + helm repo add bitnami https://charts.bitnami.com/bitnami + helm dependency build $(PROMETHEUS_PREFECT_EXPORTER_CHART_PATH) + +.PHONY: buildserver +buildserver: ## Build Server Helm dependencies + helm repo add bitnami https://charts.bitnami.com/bitnami + helm dependency build $(SERVER_CHART_PATH) + +.PHONY: buildworker +buildworker: ## Build Worker Helm dependencies + helm repo add bitnami https://charts.bitnami.com/bitnami + helm dependency build $(WORKER_CHART_PATH) + +.PHONY: buildall +buildall: buildworker buildserver buildprom + +.PHONY: helmtest +helmtest: ## Run Helm unittest + ./scripts/helm_unittest.sh diff --git a/README.md b/README.md index 5e439053..6d199c0f 100644 --- a/README.md +++ b/README.md @@ -166,18 +166,26 @@ If you are using the subchart deployed database with persistence enabled, it is Contributions to the Prefect Helm Charts are always welcome! We welcome your help - whether it's adding new functionality, echo 'eval "$(~/.local/bin/mise activate zsh)"' >> ~/.zshrctweaking documentation, or anything in between. In order to successfully contribute, you'll need to fork this repository and commit changes to your local prefect-helm repo. You can then open a PR against this upstream repo that the team will review! -To get started, ensure you have the required dependencies installed: +To get started, install the required dependencies by running: ```shell -mise install +make all ``` -Be sure to run `pre-commit install` before starting any development. [`pre-commit`](https://pre-commit.com/) -will help catch simple issues before committing. +This will install tools like `helm`, `pre-commit`, and `mise`. -### Documentation +You'll also need to install dependencies for the charts you're working on. You can use the following commands to install dependencies for each chart: -Please make sure that your changes have been linted & the chart documentation has been updated. The easiest way to accomplish this is by installing [`pre-commit`](https://pre-commit.com/). +```shell +# Server Chart +make buildserver +# Worker Chart +make buildworker +# Prometheus Prefect Exporter Chart +make buildprom +# All Charts +make buildall +``` ### Testing & validation @@ -190,7 +198,7 @@ Refer to the `helm-unittest` repository for more information. The following helper script will run the tests via the `helm-unittest` Docker image in case you don't have the binary installed locally: ```shell -./scripts/helm_unittest.sh +make helmtest ``` When `helm-unittest` is available via the [`mise` registry](https://mise.jdx.dev/registry.html), we'll add it to `.mise.toml`