From bbfcfaaeba83a4e79e8779bb37e8459712296bc4 Mon Sep 17 00:00:00 2001 From: Rahul Kumar Date: Wed, 10 Apr 2024 17:57:32 +0530 Subject: [PATCH 01/14] acquire access token before expiry --- .github/actions/create-azure-resources/action.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/actions/create-azure-resources/action.yaml b/.github/actions/create-azure-resources/action.yaml index bb41581f39..6bfc6524b7 100644 --- a/.github/actions/create-azure-resources/action.yaml +++ b/.github/actions/create-azure-resources/action.yaml @@ -31,3 +31,9 @@ runs: - name: Create additional environment variables shell: bash run: echo "subscription_id=${{ inputs.subscription-id }}" >> $GITHUB_ENV + + - name: Acquire access tokens + uses: azure/cli@v2 + with: + inlineScript: | + az account get-access-token --scope https://management.azure.com/.default From 7ec78cdc5b6f2d0e7b1bc84652a856e5b2f9cfba Mon Sep 17 00:00:00 2001 From: Rahul Kumar Date: Wed, 10 Apr 2024 17:58:49 +0530 Subject: [PATCH 02/14] temp change to trigger test --- assets/aml-benchmark/components/dataset_downloader/spec.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assets/aml-benchmark/components/dataset_downloader/spec.yaml b/assets/aml-benchmark/components/dataset_downloader/spec.yaml index e996cabca1..0d9b0066be 100644 --- a/assets/aml-benchmark/components/dataset_downloader/spec.yaml +++ b/assets/aml-benchmark/components/dataset_downloader/spec.yaml @@ -4,7 +4,7 @@ type: command name: dataset_downloader display_name: Dataset Downloader description: Downloads the dataset onto blob store. -version: 0.0.3 +version: 0.0.3.test1 inputs: dataset_name: From 63c0631326ab99216aa51d325ab252de78d74b64 Mon Sep 17 00:00:00 2001 From: Rahul Kumar Date: Wed, 10 Apr 2024 18:32:32 +0530 Subject: [PATCH 03/14] add requested url --- .github/actions/create-azure-resources/action.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/actions/create-azure-resources/action.yaml b/.github/actions/create-azure-resources/action.yaml index 6bfc6524b7..bb8ef42846 100644 --- a/.github/actions/create-azure-resources/action.yaml +++ b/.github/actions/create-azure-resources/action.yaml @@ -36,4 +36,5 @@ runs: uses: azure/cli@v2 with: inlineScript: | + az account get-access-token --scope https://management.azure.com az account get-access-token --scope https://management.azure.com/.default From dbe68e974c5d3068121b2a48cd60d6db7d700d29 Mon Sep 17 00:00:00 2001 From: Rahul Kumar Date: Wed, 10 Apr 2024 18:56:19 +0530 Subject: [PATCH 04/14] print the current time as well --- .github/actions/create-azure-resources/action.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/create-azure-resources/action.yaml b/.github/actions/create-azure-resources/action.yaml index bb8ef42846..097e9c4913 100644 --- a/.github/actions/create-azure-resources/action.yaml +++ b/.github/actions/create-azure-resources/action.yaml @@ -36,5 +36,5 @@ runs: uses: azure/cli@v2 with: inlineScript: | - az account get-access-token --scope https://management.azure.com + echo "Current time: $(date -u +"%Y-%m-%dT%H:%M:%SZ")" az account get-access-token --scope https://management.azure.com/.default From ce64c45b6f84d442c289c77bfceff740b08257a0 Mon Sep 17 00:00:00 2001 From: Rahul Kumar Date: Wed, 10 Apr 2024 19:18:25 +0530 Subject: [PATCH 05/14] update when to get token --- .github/actions/create-azure-resources/action.yaml | 7 ------- .github/workflows/assets-test.yaml | 4 +++- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/.github/actions/create-azure-resources/action.yaml b/.github/actions/create-azure-resources/action.yaml index 097e9c4913..bb41581f39 100644 --- a/.github/actions/create-azure-resources/action.yaml +++ b/.github/actions/create-azure-resources/action.yaml @@ -31,10 +31,3 @@ runs: - name: Create additional environment variables shell: bash run: echo "subscription_id=${{ inputs.subscription-id }}" >> $GITHUB_ENV - - - name: Acquire access tokens - uses: azure/cli@v2 - with: - inlineScript: | - echo "Current time: $(date -u +"%Y-%m-%dT%H:%M:%SZ")" - az account get-access-token --scope https://management.azure.com/.default diff --git a/.github/workflows/assets-test.yaml b/.github/workflows/assets-test.yaml index d91f4113d5..27f5e96923 100644 --- a/.github/workflows/assets-test.yaml +++ b/.github/workflows/assets-test.yaml @@ -140,7 +140,9 @@ jobs: scripts-setup-dir: ${{ env.scripts_setup_dir }} - name: Test asset - run: python -u $scripts_test_dir/test_assets.py -i "${{ matrix.asset_config_path }}" -a $asset_config_filename -p $scripts_test_dir/requirements.txt -r $GITHUB_WORKSPACE/$pytest_reports + run: | + az account get-access-token --scope https://management.azure.com/.default + python -u $scripts_test_dir/test_assets.py -i "${{ matrix.asset_config_path }}" -a $asset_config_filename -p $scripts_test_dir/requirements.txt -r $GITHUB_WORKSPACE/$pytest_reports - name: Upload test results uses: actions/upload-artifact@v3 From d61e403417f6708f611eccf213154e4fd83bcc89 Mon Sep 17 00:00:00 2001 From: Rahul Kumar Date: Wed, 10 Apr 2024 21:37:20 +0530 Subject: [PATCH 06/14] Revert "temp change to trigger test" This reverts commit 7ec78cdc5b6f2d0e7b1bc84652a856e5b2f9cfba. --- assets/aml-benchmark/components/dataset_downloader/spec.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assets/aml-benchmark/components/dataset_downloader/spec.yaml b/assets/aml-benchmark/components/dataset_downloader/spec.yaml index 0d9b0066be..e996cabca1 100644 --- a/assets/aml-benchmark/components/dataset_downloader/spec.yaml +++ b/assets/aml-benchmark/components/dataset_downloader/spec.yaml @@ -4,7 +4,7 @@ type: command name: dataset_downloader display_name: Dataset Downloader description: Downloads the dataset onto blob store. -version: 0.0.3.test1 +version: 0.0.3 inputs: dataset_name: From 1367d1f9dd861ccffadfa5bf5c700929baa53297 Mon Sep 17 00:00:00 2001 From: Rahul Kumar Date: Wed, 10 Apr 2024 23:09:14 +0530 Subject: [PATCH 07/14] prevent logging token --- .github/workflows/assets-test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/assets-test.yaml b/.github/workflows/assets-test.yaml index 27f5e96923..77f876ec3a 100644 --- a/.github/workflows/assets-test.yaml +++ b/.github/workflows/assets-test.yaml @@ -141,7 +141,7 @@ jobs: - name: Test asset run: | - az account get-access-token --scope https://management.azure.com/.default + az account get-access-token --scope https://management.azure.com/.default --output none python -u $scripts_test_dir/test_assets.py -i "${{ matrix.asset_config_path }}" -a $asset_config_filename -p $scripts_test_dir/requirements.txt -r $GITHUB_WORKSPACE/$pytest_reports - name: Upload test results From 3336fe4a612815c4099e130648cafa15bf7ddfce Mon Sep 17 00:00:00 2001 From: Rahul Kumar Date: Wed, 10 Apr 2024 23:10:30 +0530 Subject: [PATCH 08/14] temp bump version to trigger test --- .../components/compute_performance_metrics/spec.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assets/aml-benchmark/components/compute_performance_metrics/spec.yaml b/assets/aml-benchmark/components/compute_performance_metrics/spec.yaml index 018c6e28e5..3fd71e4dfd 100644 --- a/assets/aml-benchmark/components/compute_performance_metrics/spec.yaml +++ b/assets/aml-benchmark/components/compute_performance_metrics/spec.yaml @@ -4,7 +4,7 @@ type: command name: compute_performance_metrics display_name: Compute Performance Metrics description: Performs performance metric post processing using data from a model inference run. -version: 0.0.3 +version: 0.0.3.dev1 is_deterministic: true inputs: From a5616763286faeefd2c9bb8ac2719bf1cd0186be Mon Sep 17 00:00:00 2001 From: Rahul Kumar Date: Wed, 10 Apr 2024 23:40:26 +0530 Subject: [PATCH 09/14] Revert "prevent logging token" This reverts commit 1367d1f9dd861ccffadfa5bf5c700929baa53297. --- .github/workflows/assets-test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/assets-test.yaml b/.github/workflows/assets-test.yaml index 77f876ec3a..27f5e96923 100644 --- a/.github/workflows/assets-test.yaml +++ b/.github/workflows/assets-test.yaml @@ -141,7 +141,7 @@ jobs: - name: Test asset run: | - az account get-access-token --scope https://management.azure.com/.default --output none + az account get-access-token --scope https://management.azure.com/.default python -u $scripts_test_dir/test_assets.py -i "${{ matrix.asset_config_path }}" -a $asset_config_filename -p $scripts_test_dir/requirements.txt -r $GITHUB_WORKSPACE/$pytest_reports - name: Upload test results From 24fcf3bfcf632611e8b4e66abb20cdb78dc3c0ef Mon Sep 17 00:00:00 2001 From: Rahul Kumar Date: Thu, 11 Apr 2024 00:20:59 +0530 Subject: [PATCH 10/14] try switching to resource --- .github/workflows/assets-test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/assets-test.yaml b/.github/workflows/assets-test.yaml index 27f5e96923..f24d6cae48 100644 --- a/.github/workflows/assets-test.yaml +++ b/.github/workflows/assets-test.yaml @@ -141,7 +141,7 @@ jobs: - name: Test asset run: | - az account get-access-token --scope https://management.azure.com/.default + az account get-access-token --resource https://management.azure.com python -u $scripts_test_dir/test_assets.py -i "${{ matrix.asset_config_path }}" -a $asset_config_filename -p $scripts_test_dir/requirements.txt -r $GITHUB_WORKSPACE/$pytest_reports - name: Upload test results From 2bf03103c96ccff878b0fb894a95fa826c08920a Mon Sep 17 00:00:00 2001 From: Rahul Kumar Date: Thu, 11 Apr 2024 11:00:11 +0530 Subject: [PATCH 11/14] try fetching OID token every 4 mins --- .github/workflows/assets-test.yaml | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/.github/workflows/assets-test.yaml b/.github/workflows/assets-test.yaml index f24d6cae48..3c2907dcd9 100644 --- a/.github/workflows/assets-test.yaml +++ b/.github/workflows/assets-test.yaml @@ -138,11 +138,21 @@ jobs: tenant-id: ${{ secrets.AZURE_TENANT_ID }} subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} scripts-setup-dir: ${{ env.scripts_setup_dir }} - - - name: Test asset + + - name: Fetch OID token every 4 mins run: | - az account get-access-token --resource https://management.azure.com - python -u $scripts_test_dir/test_assets.py -i "${{ matrix.asset_config_path }}" -a $asset_config_filename -p $scripts_test_dir/requirements.txt -r $GITHUB_WORKSPACE/$pytest_reports + while true; do + token_request=$ACTIONS_ID_TOKEN_REQUEST_TOKEN + token_uri=$ACTIONS_ID_TOKEN_REQUEST_URL + token=$(curl -H "Authorization: bearer $token_request" "${token_uri}&audience=api://AzureADTokenExchange" | jq .value -r) + az login --service-principal -u ${{ secrets.AZURE_CLIENT_ID }} -t ${{ secrets.AZURE_TENANT_ID }} --federated-token $token --output none + + # Sleep for 4 minutes + sleep 240 + done & + + - name: Test asset + run: python -u $scripts_test_dir/test_assets.py -i "${{ matrix.asset_config_path }}" -a $asset_config_filename -p $scripts_test_dir/requirements.txt -r $GITHUB_WORKSPACE/$pytest_reports - name: Upload test results uses: actions/upload-artifact@v3 From fe49369e7de48fb80a3476155c76f252dba3bb74 Mon Sep 17 00:00:00 2001 From: Rahul Kumar Date: Thu, 11 Apr 2024 11:55:03 +0530 Subject: [PATCH 12/14] Revert "temp bump version to trigger test" This reverts commit 3336fe4a612815c4099e130648cafa15bf7ddfce. --- .../components/compute_performance_metrics/spec.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assets/aml-benchmark/components/compute_performance_metrics/spec.yaml b/assets/aml-benchmark/components/compute_performance_metrics/spec.yaml index 3fd71e4dfd..018c6e28e5 100644 --- a/assets/aml-benchmark/components/compute_performance_metrics/spec.yaml +++ b/assets/aml-benchmark/components/compute_performance_metrics/spec.yaml @@ -4,7 +4,7 @@ type: command name: compute_performance_metrics display_name: Compute Performance Metrics description: Performs performance metric post processing using data from a model inference run. -version: 0.0.3.dev1 +version: 0.0.3 is_deterministic: true inputs: From ac03f56276959e8d3d1423ce8663cfaf8fa5d3be Mon Sep 17 00:00:00 2001 From: Rahul Kumar Date: Fri, 12 Apr 2024 23:43:11 +0530 Subject: [PATCH 13/14] update the step name --- .github/workflows/assets-test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/assets-test.yaml b/.github/workflows/assets-test.yaml index 3c2907dcd9..79441b94ce 100644 --- a/.github/workflows/assets-test.yaml +++ b/.github/workflows/assets-test.yaml @@ -139,7 +139,7 @@ jobs: subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} scripts-setup-dir: ${{ env.scripts_setup_dir }} - - name: Fetch OID token every 4 mins + - name: Refresh login every 4 minutes (background process) run: | while true; do token_request=$ACTIONS_ID_TOKEN_REQUEST_TOKEN From ef01ebd482b321c7398271fb57d4ace386c70365 Mon Sep 17 00:00:00 2001 From: Rahul Kumar Date: Sat, 13 Apr 2024 00:17:28 +0530 Subject: [PATCH 14/14] parse jwt to get expiry --- .github/workflows/assets-test.yaml | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/workflows/assets-test.yaml b/.github/workflows/assets-test.yaml index 79441b94ce..74249259e8 100644 --- a/.github/workflows/assets-test.yaml +++ b/.github/workflows/assets-test.yaml @@ -139,16 +139,23 @@ jobs: subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} scripts-setup-dir: ${{ env.scripts_setup_dir }} - - name: Refresh login every 4 minutes (background process) + - name: Refresh login (background process) run: | while true; do token_request=$ACTIONS_ID_TOKEN_REQUEST_TOKEN token_uri=$ACTIONS_ID_TOKEN_REQUEST_URL token=$(curl -H "Authorization: bearer $token_request" "${token_uri}&audience=api://AzureADTokenExchange" | jq .value -r) + + expiration_time=$(jq -R 'split(".") | .[1] | @base64d | fromjson | .exp' <<< $token) + expiration_time_formatted=$(date -u -d @$expiration_time +"%Y-%m-%dT%H:%M:%SZ") + current_time=$(date -u +"%Y-%m-%dT%H:%M:%SZ") + echo "Token expires at: $expiration_time_formatted, Current time: $current_time" + az login --service-principal -u ${{ secrets.AZURE_CLIENT_ID }} -t ${{ secrets.AZURE_TENANT_ID }} --federated-token $token --output none - # Sleep for 4 minutes - sleep 240 + sleep_timer=$((expiration_time - $(date +%s) - 30)) + echo "Sleeping for $sleep_timer seconds" + sleep $sleep_timer done & - name: Test asset