-
Notifications
You must be signed in to change notification settings - Fork 27
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'branch-24.10' into fix-repo-clone-submodules
- Loading branch information
Showing
22 changed files
with
832 additions
and
24 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
23 changes: 23 additions & 0 deletions
23
features/src/utils/opt/devcontainer/bin/creds/s3/generate.sh
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,23 @@ | ||
| #! /usr/bin/env bash | ||
|
|
||
| _creds_s3_generate() { | ||
| local -; | ||
| set -euo pipefail; | ||
|
|
||
| # shellcheck disable=SC1091 | ||
| . devcontainer-utils-debug-output 'devcontainer_utils_debug' 'creds-s3 creds-s3-generate'; | ||
|
|
||
| if test -z "${SCCACHE_BUCKET:-}"; then | ||
| exit 1; | ||
| fi | ||
|
|
||
| if test -n "${AWS_ROLE_ARN:-}" && gh nv-gha-aws --help >/dev/null 2>&1; then | ||
| # shellcheck disable=SC1091 | ||
| devcontainer-utils-creds-s3-gh-generate; | ||
| elif test -n "${VAULT_HOST}"; then | ||
| # shellcheck disable=SC1091 | ||
| devcontainer-utils-creds-s3-vault-generate; | ||
| fi | ||
| } | ||
|
|
||
| _creds_s3_generate "$@" <&0; |
68 changes: 68 additions & 0 deletions
68
features/src/utils/opt/devcontainer/bin/creds/s3/gh/generate.sh
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,68 @@ | ||
| #! /usr/bin/env bash | ||
|
|
||
| _creds_github_generate() { | ||
| local -; | ||
| set -euo pipefail; | ||
|
|
||
| # shellcheck disable=SC1091 | ||
| . devcontainer-utils-debug-output 'devcontainer_utils_debug' 'creds-s3 creds-s3-vault creds-s3-vault-generate'; | ||
|
|
||
| if test -z "${AWS_ROLE_ARN:-}" \ | ||
| || test -z "${SCCACHE_BUCKET:-}" \ | ||
| || ! gh nv-gha-aws --help >/dev/null 2>&1; then | ||
| exit 1; | ||
| fi | ||
|
|
||
| # Remove existing credentials in case vault declines to issue new ones. | ||
| rm -rf ~/.aws/{stamp,config,credentials}; | ||
|
|
||
| SCCACHE_REGION="${SCCACHE_REGION:-${AWS_DEFAULT_REGION:-}}"; | ||
|
|
||
| devcontainer-utils-creds-s3-persist - <<< \ | ||
| --bucket="${SCCACHE_BUCKET:-}" \ | ||
| --region="${SCCACHE_REGION:-}" ; | ||
| # Initialize the GitHub CLI with the appropriate user scopes | ||
| # shellcheck disable=SC1091 | ||
| . devcontainer-utils-init-github-cli; | ||
| # Check whether the user is in one of the allowed GitHub orgs | ||
| local allowed_orgs="${AWS_GITHUB_ORGS:-${VAULT_GITHUB_ORGS:-nvidia nv-morpheus nv-legate rapids}}"; | ||
| allowed_orgs="${allowed_orgs// /|}"; | ||
| allowed_orgs="${allowed_orgs//;/|}"; | ||
| allowed_orgs="${allowed_orgs//,/|}"; | ||
| local -ra user_orgs="($( \ | ||
| gh api user/orgs --jq '.[].login' \ | ||
| -H "Accept: application/vnd.github+json" \ | ||
| | grep --color=never -iE "(${allowed_orgs})" \ | ||
| ))"; | ||
| if test "${#user_orgs[@]}" -eq 0; then | ||
| exit 1; | ||
| fi | ||
| local org; | ||
| local generated_at; | ||
| local -a nv_gha_aws_args=( | ||
| --profile default | ||
| --output creds-file | ||
| --role-arn "${AWS_ROLE_ARN}" | ||
| --aud "${AWS_AUDIENCE:-sts.amazonaws.com}" | ||
| --duration "${AWS_S3_TTL:-${VAULT_S3_TTL:-43200}}" | ||
| --idp-url "${AWS_IDP_URL:-https://token.gha-runners.nvidia.com}" | ||
| ); | ||
| for org in "${user_orgs[@]}"; do | ||
| generated_at="$(date '+%s')"; | ||
| if gh nv-gha-aws org "${org}" "${nv_gha_aws_args[@]}" >"${HOME}/.aws/credentials" 2>>/var/log/devcontainer-utils/creds-s3.log; then | ||
| if devcontainer-utils-creds-s3-propagate 2>&1 | tee -a /var/log/devcontainer-utils/creds-s3.log; then | ||
| echo "${generated_at}" > ~/.aws/stamp; | ||
| return 0; | ||
| fi | ||
| fi | ||
| done | ||
| } | ||
| _creds_github_generate "$@" <&0; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,58 @@ | ||
| #! /usr/bin/env bash | ||
|
|
||
| _s3_cred() { | ||
| sed -n "s/$1=//p" ~/.aws/credentials 2>/dev/null; | ||
| } | ||
|
|
||
| _s3_creds_init() { | ||
| local - | ||
| set -euo pipefail; | ||
|
|
||
| # shellcheck disable=SC1091 | ||
| . devcontainer-utils-debug-output 'devcontainer_utils_debug' 'creds-s3 creds-s3-init'; | ||
|
|
||
| if type sccache >/dev/null 2>&1; then | ||
| if ! grep -qE "^$" <<< "${SCCACHE_BUCKET:-}"; then | ||
| if grep -qE "^$" <<< "${AWS_ACCESS_KEY_ID:-}" \ | ||
| && grep -qE "^$" <<< "${AWS_SECRET_ACCESS_KEY:-}" ; then | ||
| if ! grep -qE "^$" <<< "${VAULT_HOST:-${AWS_ROLE_ARN:-}}"; then | ||
| # Generate S3 creds if they don't exist (or are expired) | ||
| if devcontainer-utils-creds-s3-test \ | ||
| || devcontainer-utils-creds-s3-generate; then | ||
| # Persist creds in ~/.aws dir | ||
| devcontainer-utils-creds-s3-persist - <<< " \ | ||
| --bucket '${SCCACHE_BUCKET:-}' \ | ||
| --region '${SCCACHE_REGION:-${AWS_DEFAULT_REGION:-}}' \ | ||
| --aws-access-key-id '$(_s3_cred aws_access_key_id)' \ | ||
| --aws-session-token '$(_s3_cred aws_session_token)' \ | ||
| --aws-secret-access-key '$(_s3_cred aws_secret_access_key)' \ | ||
| "; | ||
| # Install a crontab to refresh the credentials | ||
| devcontainer-utils-creds-s3-schedule; | ||
| else | ||
| devcontainer-utils-creds-s3-persist - <<< "--no-bucket --no-region"; | ||
| fi | ||
| elif devcontainer-utils-creds-s3-test; then | ||
| devcontainer-utils-creds-s3-persist - <<< " \ | ||
| --bucket '${SCCACHE_BUCKET:-}' \ | ||
| --region '${SCCACHE_REGION:-${AWS_DEFAULT_REGION:-}}' \ | ||
| --aws-access-key-id '$(_s3_cred aws_access_key_id)' \ | ||
| --aws-session-token '$(_s3_cred aws_session_token)' \ | ||
| --aws-secret-access-key '$(_s3_cred aws_secret_access_key)' \ | ||
| "; | ||
| else | ||
| # bucket is inaccessible | ||
| devcontainer-utils-creds-s3-persist - <<< "--no-bucket --no-region"; | ||
| fi | ||
| elif ! devcontainer-utils-creds-s3-propagate; then | ||
| # bucket is inaccessible | ||
| devcontainer-utils-creds-s3-persist <<< "--no-bucket --no-region"; | ||
| fi | ||
| fi | ||
| fi | ||
| } | ||
|
|
||
| _s3_creds_init "$@"; | ||
|
|
||
| # shellcheck disable=SC1090 | ||
| . /etc/profile.d/*-devcontainer-utils.sh; |
109 changes: 109 additions & 0 deletions
109
features/src/utils/opt/devcontainer/bin/creds/s3/persist.sh
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,109 @@ | ||
| #!/usr/bin/env bash | ||
|
|
||
| # Usage: | ||
| # devcontainer-utils-creds-s3-persist [OPTION]... | ||
| # | ||
| # Set, unset, or reset the S3 bucket, region, and credentials in the environment. | ||
| # | ||
| # Boolean options: | ||
| # -h,--help print this text | ||
| # --no-bucket Unset the $SCCACHE_BUCKET environment variable for all shells. | ||
| # (default: false) | ||
| # --no-region Unset the $SCCACHE_REGION environment variable for all shells. | ||
| # (default: false) | ||
| # | ||
| # Options that require values: | ||
| # --stamp <stamp> Timestamp when the S3 credentials were generated. | ||
| # (default: none) | ||
| # --bucket <bucket> Set the $SCCACHE_BUCKET environment variable for all shells to <bucket> and persist in ~/.aws/config. | ||
| # (default: none) | ||
| # --region <region> Set the $SCCACHE_REGION environment variable for all shells to <region> and persist in ~/.aws/config. | ||
| # (default: none) | ||
| # --aws-access-key-id <id> Set the $AWS_ACCESS_KEY_ID environment variable for all shells to <id> and persist in ~/.aws/credentials. | ||
| # (default: none) | ||
| # --aws-session-token <token> Set the $AWS_SESSION_TOKEN environment variable for all shells to <token> and persist in ~/.aws/credentials. | ||
| # (default: none) | ||
| # --aws-secret-access-key <key> Set the $AWS_SECRET_ACCESS_KEY environment variable for all shells to <key> and persist in ~/.aws/credentials. | ||
| # (default: none) | ||
|
|
||
| # shellcheck disable=SC1091 | ||
| . "$(dirname "$(realpath -m "${BASH_SOURCE[0]}")")/../../update-envvars.sh"; | ||
|
|
||
| _creds_s3_persist() { | ||
| local -; | ||
| set -euo pipefail; | ||
|
|
||
| eval "$(devcontainer-utils-parse-args "$0" "$@" <&0)"; | ||
|
|
||
| # shellcheck disable=SC1091 | ||
| . devcontainer-utils-debug-output 'devcontainer_utils_debug' 'creds-s3 creds-s3-persist'; | ||
|
|
||
| # Reset envvars | ||
| reset_envvar "SCCACHE_BUCKET"; | ||
| reset_envvar "SCCACHE_REGION"; | ||
| reset_envvar "AWS_ACCESS_KEY_ID"; | ||
| reset_envvar "AWS_SESSION_TOKEN"; | ||
| reset_envvar "AWS_SECRET_ACCESS_KEY"; | ||
|
|
||
| mkdir -p ~/.aws; | ||
| rm -f ~/.aws/{config,credentials}; | ||
|
|
||
| if test -n "${stamp:-}"; then | ||
| echo "${stamp:-}" > ~/.aws/stamp; | ||
| fi | ||
|
|
||
| if ! grep -qE "^$" <<< "${no_bucket-}"; then | ||
| unset_envvar "SCCACHE_BUCKET"; | ||
| elif ! grep -qE "^$" <<< "${bucket:-}"; then | ||
| export_envvar "SCCACHE_BUCKET" "${bucket}"; | ||
| cat <<________EOF >> ~/.aws/config | ||
| bucket=${bucket:-} | ||
| ________EOF | ||
| fi | ||
|
|
||
| if ! grep -qE "^$" <<< "${no_region-}"; then | ||
| unset_envvar "SCCACHE_REGION"; | ||
| elif ! grep -qE "^$" <<< "${region:-}"; then | ||
| export_envvar "SCCACHE_REGION" "${region}"; | ||
| cat <<________EOF >> ~/.aws/config | ||
| region=${region:-} | ||
| ________EOF | ||
| fi | ||
|
|
||
| if test -f ~/.aws/config; then | ||
| cat <<________EOF > ~/.aws/config2 && mv ~/.aws/config{2,} | ||
| [default] | ||
| $(cat ~/.aws/config) | ||
| ________EOF | ||
| fi | ||
|
|
||
| if ! grep -qE "^$" <<< "${aws_access_key_id:-}"; then | ||
| cat <<________EOF >> ~/.aws/credentials | ||
| aws_access_key_id=${aws_access_key_id} | ||
| ________EOF | ||
| fi | ||
|
|
||
| if ! grep -qE "^$" <<< "${aws_secret_access_key:-}"; then | ||
| cat <<________EOF >> ~/.aws/credentials | ||
| aws_secret_access_key=${aws_secret_access_key} | ||
| ________EOF | ||
| fi | ||
|
|
||
| if ! grep -qE "^$" <<< "${aws_session_token:-}"; then | ||
| cat <<________EOF >> ~/.aws/credentials | ||
| aws_session_token=${aws_session_token} | ||
| ________EOF | ||
| fi | ||
|
|
||
| if test -f ~/.aws/credentials; then | ||
| cat <<________EOF > ~/.aws/credentials2 && mv ~/.aws/credentials{2,} | ||
| [default] | ||
| $(cat ~/.aws/credentials) | ||
| ________EOF | ||
| chmod 0600 ~/.aws/credentials; | ||
| fi | ||
| } | ||
|
|
||
| if [ "$(basename "${BASH_SOURCE[${#BASH_SOURCE[@]}-1]}")" = devcontainer-utils-creds-s3-persist ]; then | ||
| _creds_s3_persist "$@" <&0; | ||
| fi |
Oops, something went wrong.