From 80c144854cb0f54b1bb1106c9a4d9bea5b7b0116 Mon Sep 17 00:00:00 2001 From: Shwetha Gururaj Date: Fri, 6 Sep 2024 12:45:07 -0400 Subject: [PATCH 1/3] Use single env --- .github/ops-files/diego-cell-instances.yml | 2 +- .../workflows/tests-integration-reusable.yml | 111 ++----------- .github/workflows/tests-integration.yml | 153 +++++++++++++++++- integration/helpers/name_generator.go | 5 + integration/helpers/route.go | 10 ++ .../helpers/servicebrokerstub/app_deploy.go | 15 +- .../shared/isolated/isolated_suite_test.go | 8 + .../global/update_buildpack_command_test.go | 2 +- integration/v7/isolated/app_command_test.go | 4 +- .../v7/isolated/unmap_route_command_test.go | 2 +- 10 files changed, 208 insertions(+), 104 deletions(-) diff --git a/.github/ops-files/diego-cell-instances.yml b/.github/ops-files/diego-cell-instances.yml index 87d715d696c..19ea43d8077 100644 --- a/.github/ops-files/diego-cell-instances.yml +++ b/.github/ops-files/diego-cell-instances.yml @@ -1,4 +1,4 @@ --- - type: replace path: /instance_groups/name=diego-cell/instances - value: 3 + value: 4 diff --git a/.github/workflows/tests-integration-reusable.yml b/.github/workflows/tests-integration-reusable.yml index f81acdb4282..cf91d0beadc 100644 --- a/.github/workflows/tests-integration-reusable.yml +++ b/.github/workflows/tests-integration-reusable.yml @@ -19,12 +19,9 @@ on: name: required: true type: string - pool-name: - type: string - default: ${{ vars.SHEPHERD_POOL_NAME }} - pool-namespace: + lease-id: + required: true type: string - default: 'official' gitRef: type: string default: ${{github.event.workflow_run.head_sha}} @@ -50,51 +47,12 @@ jobs: ref: release-candidate path: cf-acceptance-tests - - name: Checkout cf-deployment - uses: actions/checkout@v4 - with: - repository: cloudfoundry/cf-deployment - path: cf-deployment - - name: Checkout CF deployment tasks uses: actions/checkout@v4 with: repository: cloudfoundry/cf-deployment-concourse-tasks path: cf-deployment-concourse-tasks - - id: claim-env - name: Claim Environment - env: - account_token: ${{ secrets.SHEPHERD_SERVICE_ACCOUNT_TOKEN }} - pool_name: ${{ inputs.pool-name }} - pool_namespace: ${{ inputs.pool-namespace }} - run: | - shepherd login service-account ${account_token} - - echo "shepherd create lease --duration 8h --pool ${pool_name} --pool-namespace ${pool_namespace} --namespace tas-devex --description 'CLI GHA'" - lease_id=$(shepherd create lease --duration 8h --pool ${pool_name} --pool-namespace ${pool_namespace} --namespace tas-devex --json | jq -r .id) - # Give sometime for the lease to complete. Shepherd may take upto an 3 hours to create an env - # if the pool is empty. - count=0 - while [ $count -lt 360 ] ; do - sleep 30 - status=$(shepherd get lease ${lease_id} --namespace tas-devex --json | jq -r .status) - if [ $status == "LEASED" ] ; then - shepherd get lease ${lease_id} --namespace tas-devex --json | jq .output > metadata.json - break - elif [ $status == "FAILED" -o $status == "EXPIRED" ] ; then - echo "There was an error obtaining the lease. Lease status is ${status}." - exit 1 - else - echo "Waiting for environment to be ready. Lease status is ${status}." - fi - count=$(($count+1)) - done - - env_name=$(jq -r .name metadata.json) - cat metadata.json | jq -r '.name' - echo "lease-id=$lease_id" >> "${GITHUB_OUTPUT}" - - name: Set Up Go uses: actions/setup-go@v5 with: @@ -102,6 +60,8 @@ jobs: check-latest: true - name: Install Tools + env: + account_token: ${{ secrets.SHEPHERD_SERVICE_ACCOUNT_TOKEN }} run: | go version @@ -109,19 +69,19 @@ jobs: then install_location=/usr/bin bbl_artifact=bbl-v8.4.110_windows.exe - bosh_cli_artifact=bosh-cli-7.0.1-windows-amd64.exe + bosh_cli_artifact=bosh-cli-7.7.2-windows-amd64.exe credhub_artifact=credhub-windows-2.9.4.tgz else install_location=/usr/local/bin bbl_artifact=bbl-v8.4.110_linux_x86-64 - bosh_cli_artifact=bosh-cli-7.0.1-linux-amd64 + bosh_cli_artifact=bosh-cli-7.7.2-linux-amd64 credhub_artifact=credhub-linux-2.9.4.tgz fi curl https://github.com/cloudfoundry/bosh-bootloader/releases/download/v8.4.110/${bbl_artifact} --silent --location --output $install_location/bbl chmod +x $install_location/bbl bbl --version - curl https://s3.amazonaws.com/bosh-cli-artifacts/$bosh_cli_artifact --silent --output $install_location/bosh --location + curl https://github.com/cloudfoundry/bosh-cli/releases/download/v7.7.2/$bosh_cli_artifact --silent --output $install_location/bosh --location chmod +x $install_location/bosh bosh --version @@ -134,50 +94,14 @@ jobs: apt-get update apt-get install -y build-essential unzip - - name: Upload latest CAPI release - env: - capi_release_version: ${{ vars.CAPI_RELEASE_VERSION }} - run: | - if [ -z "$capi_release_version" ] - then - capi_release_version=$(curl -s https://api.github.com/repos/cloudfoundry/capi-release/releases/latest | jq -r .tag_name) - fi - - echo "Latest CAPI release is $capi_release_version" - - eval "$(bbl print-env --metadata-file metadata.json)" - env_name=$(jq -r .name metadata.json) - jq -r .bosh.jumpbox_private_key metadata.json > /tmp/${env_name}.priv - - bosh upload-release "https://bosh.io/d/github.com/cloudfoundry/capi-release?v=$capi_release_version" - - - name: Deploy Isolation Segment and OIDC Provider - run: | - env_name=$(jq -r .name metadata.json) - jq -r .bosh.jumpbox_private_key metadata.json > /tmp/${env_name}.priv - eval "$(bbl print-env --metadata-file metadata.json)" - - # deploy - bosh -d cf manifest > /tmp/manifest.yml - bosh interpolate /tmp/manifest.yml \ - -o cf-deployment/operations/use-internal-lookup-for-route-services.yml \ - -o cf-deployment/operations/add-persistent-isolation-segment-diego-cell.yml \ - -o .github/ops-files/use-latest-capi.yml \ - -o .github/ops-files/add-oidc-provider.yml \ - -o .github/ops-files/add-uaa-client-credentials.yml \ - -o .github/ops-files/diego-cell-instances.yml \ - -v client-secret="${{ secrets.CLIENT_SECRET }}" \ - > ./director.yml - - bosh -d cf deploy director.yml -n - echo "Deployed CAPI version:" - bosh -d cf releases | grep capi + shepherd login service-account ${account_token} + shepherd get lease ${{ inputs.lease-id }} --namespace tas-devex --json | jq .output > metadata.json - name: Add CATS config if: ${{ inputs.name == 'cats' }} run: | set -eu - + ENV=$(jq -r .name metadata.json) API="$(jq -r .cf.api_url metadata.json)" DOMAIN=$(echo $API | sed "s/^api\.//") @@ -207,6 +131,7 @@ jobs: "include_container_networking": true, "include_detect": true, "include_docker": true, + "include_cnb": true, "include_internet_dependent": true, "include_isolation_segments": true, "isolation_segment_name": "persistent_isolation_segment", @@ -237,6 +162,7 @@ jobs: cf api ${API} --skip-ssl-validation cf auth cf enable-feature-flag diego_docker + cf enable-feature-flag diego_cnb cf enable-feature-flag service_instance_sharing - name: Run CATS Tests @@ -282,6 +208,7 @@ jobs: export GOPATH=$PWD/go export PATH="$GOPATH/bin:$PATH" export PATH="$PWD/out:$PATH" + export CF_INT_TEST_NAME="int" make build @@ -312,6 +239,7 @@ jobs: export GOPATH=$PWD/go export PATH="$GOPATH/bin:$PATH" export PATH="$PWD/out:$PATH" + export CF_INT_TEST_NAME="cc" make build @@ -319,13 +247,4 @@ jobs: cf api ${CF_INT_API} --skip-ssl-validation cf auth - make integration-tests-full-ci - - - name: Unclaim environment - if: always() - env: - account_token: ${{ secrets.SHEPHERD_SERVICE_ACCOUNT_TOKEN }} - run: | - shepherd login service-account ${account_token} - set -x - shepherd delete lease ${{ steps.claim-env.outputs.lease-id }} --namespace tas-devex + make integration-tests-full-ci \ No newline at end of file diff --git a/.github/workflows/tests-integration.yml b/.github/workflows/tests-integration.yml index 4da7454fc4e..dc2624d9115 100644 --- a/.github/workflows/tests-integration.yml +++ b/.github/workflows/tests-integration.yml @@ -16,13 +16,11 @@ on: - run-cats-cf-env push: tags: - - "v9.*" - "v8.*" - "v7.*" pull_request_target: branches: - main - - v9 - v8 - v7 paths-ignore: @@ -64,11 +62,140 @@ jobs: check-latest: true - name: Run Units run: make units + + claim-env: + name: Claim and Prep Environment + outputs: + leaseid: ${{steps.claim.outputs.leaseid}} + defaults: + run: + shell: bash + runs-on: ubuntu-latest + container: us-west2-docker.pkg.dev/shepherd-268822/shepherd2/concourse-resource:latest + needs: + - get-sha + - units + steps: + - name: Checkout cli + uses: actions/checkout@v4 + with: + ref: ${{needs.get-sha.outputs.gitRef}} + + - name: claim + id: claim + env: + account_token: ${{ secrets.SHEPHERD_SERVICE_ACCOUNT_TOKEN }} + pool_name: ${{ vars.SHEPHERD_POOL_NAME }} + pool_namespace: official + run: | + shepherd login service-account ${account_token} + + echo "shepherd create lease --duration 8h --pool ${pool_name} --pool-namespace ${pool_namespace} --namespace tas-devex --description 'CLI GHA'" + lease_id=$(shepherd create lease --duration 8h --pool ${pool_name} --pool-namespace ${pool_namespace} --namespace tas-devex --json | jq -r .id) + + # Give sometime for the lease to complete. Shepherd may take upto an 3 hours to create an env + # if the pool is empty. + count=0 + while [ $count -lt 360 ] ; do + sleep 30 + status=$(shepherd get lease ${lease_id} --namespace tas-devex --json | jq -r .status) + if [ $status == "LEASED" ] ; then + shepherd get lease ${lease_id} --namespace tas-devex --json | jq .output > metadata.json + break + elif [ $status == "FAILED" -o $status == "EXPIRED" ] ; then + echo "There was an error obtaining the lease. Lease status is ${status}." + exit 1 + else + echo "Waiting for environment to be ready. Lease status is ${status}." + fi + count=$(($count+1)) + done + + env_name=$(jq -r .name metadata.json) + echo "env name is ${env_name}" + echo "leaseid=${lease_id}" >> "${GITHUB_OUTPUT}" + + cf_deployment_version=$(jq -r '."cf-deployment_version"' metadata.json) + echo "cf_deployment_version is ${cf_deployment_version}" + echo "cf_deployment_version=${cf_deployment_version}" >> "${GITHUB_OUTPUT}" + + - name: Set Up Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + check-latest: true + + - name: Install Tools + run: | + go version + + install_location=/usr/local/bin + bbl_artifact=bbl-v8.4.110_linux_x86-64 + bosh_cli_artifact=bosh-cli-7.7.2-linux-amd64 + + curl https://github.com/cloudfoundry/bosh-bootloader/releases/download/v8.4.110/${bbl_artifact} --silent --location --output $install_location/bbl + chmod +x $install_location/bbl + bbl --version + + curl https://github.com/cloudfoundry/bosh-cli/releases/download/v7.7.2/$bosh_cli_artifact --silent --output $install_location/bosh --location + chmod +x $install_location/bosh + bosh --version + + apt-get update + apt-get install -y build-essential unzip + + - name: Upload latest CAPI release + env: + capi_release_version: ${{ vars.CAPI_RELEASE_VERSION }} + run: | + if [ -z "$capi_release_version" ] + then + capi_release_version=$(curl -s https://api.github.com/repos/cloudfoundry/capi-release/releases/latest | jq -r .tag_name) + fi + + echo "Latest CAPI release is $capi_release_version" + + eval "$(bbl print-env --metadata-file metadata.json)" + env_name=$(jq -r .name metadata.json) + jq -r .bosh.jumpbox_private_key metadata.json > /tmp/${env_name}.priv + + bosh upload-release "https://bosh.io/d/github.com/cloudfoundry/capi-release?v=$capi_release_version" + + - name: Checkout cf-deployment + uses: actions/checkout@v4 + with: + repository: cloudfoundry/cf-deployment + path: cf-deployment + ref: ${{steps.claim.outputs.cf_deployment_version}} + + - name: Deploy Isolation Segment and OIDC Provider + run: | + env_name=$(jq -r .name metadata.json) + jq -r .bosh.jumpbox_private_key metadata.json > /tmp/${env_name}.priv + eval "$(bbl print-env --metadata-file metadata.json)" + + # deploy + bosh -d cf manifest > /tmp/manifest.yml + bosh interpolate /tmp/manifest.yml \ + -o cf-deployment/operations/use-internal-lookup-for-route-services.yml \ + -o cf-deployment/operations/add-persistent-isolation-segment-diego-cell.yml \ + -o .github/ops-files/use-latest-capi.yml \ + -o .github/ops-files/add-oidc-provider.yml \ + -o .github/ops-files/add-uaa-client-credentials.yml \ + -o .github/ops-files/diego-cell-instances.yml \ + -v client-secret="${{ secrets.CLIENT_SECRET }}" \ + > ./director.yml + + bosh -d cf deploy director.yml -n + echo "Deployed CAPI version:" + bosh -d cf releases | grep capi + run-integration-tests-cf-env: name: Integration tests needs: - get-sha - units + - claim-env if: ${{ github.event_name != 'workflow_dispatch' || inputs.workflow == 'all' || inputs.workflow == 'run-integration-tests-cf-env' }} uses: ./.github/workflows/tests-integration-reusable.yml with: @@ -76,6 +203,7 @@ jobs: os: ubuntu-latest name: Integration gitRef: ${{needs.get-sha.outputs.gitRef}} + lease-id: ${{ needs.claim-env.outputs.leaseid }} secrets: inherit run-integration-tests-cf-env-with-client-creds: @@ -83,6 +211,7 @@ jobs: needs: - get-sha - units + - claim-env if: ${{ github.event_name != 'workflow_dispatch' || inputs.workflow == 'all' || inputs.workflow == 'run-integration-tests-cf-env-with-client-creds' }} uses: ./.github/workflows/tests-integration-reusable.yml with: @@ -90,12 +219,14 @@ jobs: os: ubuntu-latest name: Integration client creds gitRef: ${{needs.get-sha.outputs.gitRef}} + lease-id: ${{ needs.claim-env.outputs.leaseid }} secrets: inherit run-cats-cf-env: name: CATS needs: - get-sha + - claim-env - run-integration-tests-cf-env - run-integration-tests-cf-env-with-client-creds if: ${{ github.event_name != 'workflow_dispatch' || inputs.workflow == 'all' || inputs.workflow == 'run-cats-cf-env' }} @@ -105,4 +236,22 @@ jobs: os: ubuntu-latest name: cats gitRef: ${{needs.get-sha.outputs.gitRef}} + lease-id: ${{ needs.claim-env.outputs.leaseid }} secrets: inherit + + unclaim-env: + name: Unclaim environment + runs-on: ubuntu-latest + container: us-west2-docker.pkg.dev/shepherd-268822/shepherd2/concourse-resource:latest + needs: + - claim-env + - run-cats-cf-env + if: always() + steps: + - name: unclaim + env: + account_token: ${{ secrets.SHEPHERD_SERVICE_ACCOUNT_TOKEN }} + run: | + shepherd login service-account ${account_token} + set -x + shepherd delete lease ${{ needs.claim-env.outputs.leaseid }} --namespace tas-devex diff --git a/integration/helpers/name_generator.go b/integration/helpers/name_generator.go index 3e54cea0a51..c6fcfeed525 100644 --- a/integration/helpers/name_generator.go +++ b/integration/helpers/name_generator.go @@ -2,6 +2,7 @@ package helpers import ( "fmt" + "os" "sort" "strings" @@ -152,6 +153,10 @@ func NewPath() string { // PrefixedRandomName provides a random name with structure "namePrefix-randomstring" func PrefixedRandomName(namePrefix string) string { + testName := os.Getenv("CF_INT_TEST_NAME") + if len(testName) > 0 { + namePrefix += "-" + testName + } return namePrefix + "-" + RandomName() } diff --git a/integration/helpers/route.go b/integration/helpers/route.go index 53eaf9d8621..d9ad43c9b12 100644 --- a/integration/helpers/route.go +++ b/integration/helpers/route.go @@ -2,6 +2,7 @@ package helpers import ( "fmt" + "os" "path" "regexp" "strings" @@ -142,3 +143,12 @@ func (r Route) GUID() string { return routeReceiver.Routes[0].GUID } + +func GetPort() int { + port := 1024 + testName := os.Getenv("CF_INT_TEST_NAME") + if testName == "cc" { + port = 1025 + } + return port +} diff --git a/integration/helpers/servicebrokerstub/app_deploy.go b/integration/helpers/servicebrokerstub/app_deploy.go index aee939be88a..be4c7abcc63 100644 --- a/integration/helpers/servicebrokerstub/app_deploy.go +++ b/integration/helpers/servicebrokerstub/app_deploy.go @@ -15,16 +15,25 @@ import ( . "github.com/onsi/gomega" ) +var appOrg = "fakeservicebroker" + const ( appNamePrefix = "hydrabroker" - appOrg = "fakeservicebroker" appSpace = "integration" defaultMemoryLimit = "32M" pathToApp = "../../assets/hydrabroker" ) +var once sync.Once var mutex sync.Mutex +func initialize() { + testName := os.Getenv("CF_INT_TEST_NAME") + if len(testName) > 0 { + appOrg += "-" + testName + } +} + func ensureAppIsDeployed() { if !appResponds() { ensureAppIsPushed() @@ -73,6 +82,10 @@ func ensureAppIsPushed() { } } + once.Do(func() { + initialize() + }) + // mutex protects from goroutines, and we retry later to protect from other test processes mutex.Lock() defer mutex.Unlock() diff --git a/integration/shared/isolated/isolated_suite_test.go b/integration/shared/isolated/isolated_suite_test.go index 8a01781a206..b74a7689d70 100644 --- a/integration/shared/isolated/isolated_suite_test.go +++ b/integration/shared/isolated/isolated_suite_test.go @@ -1,6 +1,8 @@ package isolated import ( + "flag" + "fmt" "testing" "code.cloudfoundry.org/cli/integration/helpers/commonisolated" @@ -15,10 +17,16 @@ var ( // Per Test Level homeDir string + myFlag string ) +func init() { + flag.StringVar(&myFlag, "myFlag", "defaultvalue", "myFlag is used to control my behavior") +} func TestIsolated(t *testing.T) { commonisolated.CommonTestIsolated(t) + // flag.StringVar(&myFlag, "myFlag", "defaultvalue", "myFlag is used to control my behavior") + fmt.Println("myFlag =========================", myFlag) } var _ = commonisolated.CommonGinkgoSetup( diff --git a/integration/v7/global/update_buildpack_command_test.go b/integration/v7/global/update_buildpack_command_test.go index 789473d2043..8409fd6c447 100644 --- a/integration/v7/global/update_buildpack_command_test.go +++ b/integration/v7/global/update_buildpack_command_test.go @@ -105,7 +105,7 @@ var _ = Describe("update-buildpack command", func() { Eventually(createSession).Should(Exit(0)) }) - listSession := helpers.CF("buildpacks") + listSession := helpers.CF("buildpacks").Wait() Eventually(listSession).Should(Say(helpers.BuildpacksOutputRegex(helpers.BuildpackFields{ Name: buildpackName, Stack: stacks[0]}))) Eventually(listSession).Should(Say(helpers.BuildpacksOutputRegex(helpers.BuildpackFields{Name: buildpackName}))) diff --git a/integration/v7/isolated/app_command_test.go b/integration/v7/isolated/app_command_test.go index 4e09ca52292..c78a72a97ba 100644 --- a/integration/v7/isolated/app_command_test.go +++ b/integration/v7/isolated/app_command_test.go @@ -296,8 +296,8 @@ applications: Eventually(func(g Gomega) { session := helpers.CF("app", appName).Wait() - g.Expect(session).Should(Say("Active deployment with status CANCELING")) - g.Expect(session).Should(Say("strategy: canary")) + g.Expect(session).Should(Say("instances:\\s+1/1")) + g.Expect(session).ShouldNot(Say("instances:\\s+1/1")) g.Expect(session).Should(Exit(0)) }).Should(Succeed()) }) diff --git a/integration/v7/isolated/unmap_route_command_test.go b/integration/v7/isolated/unmap_route_command_test.go index 13bd47c4b7e..7143031a98b 100644 --- a/integration/v7/isolated/unmap_route_command_test.go +++ b/integration/v7/isolated/unmap_route_command_test.go @@ -117,7 +117,7 @@ var _ = Describe("unmap-route command", func() { }) When("it's a TCP route", func() { BeforeEach(func() { - port = 1024 + port = helpers.GetPort() tcpRoute = helpers.NewTCPRoute(spaceName, tcpDomain.Name, port) session := helpers.CF("map-route", appName, tcpDomain.Name, "--port", fmt.Sprintf("%d", tcpRoute.Port)) Eventually(session).Should(Exit(0)) From 26e0cb25ea8e595760cd66a90df30074da21508b Mon Sep 17 00:00:00 2001 From: Shwetha Gururaj Date: Thu, 3 Oct 2024 15:34:54 -0400 Subject: [PATCH 2/3] Undo changes --- integration/shared/isolated/isolated_suite_test.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/integration/shared/isolated/isolated_suite_test.go b/integration/shared/isolated/isolated_suite_test.go index b74a7689d70..8a01781a206 100644 --- a/integration/shared/isolated/isolated_suite_test.go +++ b/integration/shared/isolated/isolated_suite_test.go @@ -1,8 +1,6 @@ package isolated import ( - "flag" - "fmt" "testing" "code.cloudfoundry.org/cli/integration/helpers/commonisolated" @@ -17,16 +15,10 @@ var ( // Per Test Level homeDir string - myFlag string ) -func init() { - flag.StringVar(&myFlag, "myFlag", "defaultvalue", "myFlag is used to control my behavior") -} func TestIsolated(t *testing.T) { commonisolated.CommonTestIsolated(t) - // flag.StringVar(&myFlag, "myFlag", "defaultvalue", "myFlag is used to control my behavior") - fmt.Println("myFlag =========================", myFlag) } var _ = commonisolated.CommonGinkgoSetup( From 0f2704da8f5186b7744f6581913ea5a46e71b4cf Mon Sep 17 00:00:00 2001 From: Shwetha Gururaj Date: Thu, 3 Oct 2024 16:57:15 -0400 Subject: [PATCH 3/3] Update .github/workflows/tests-integration-reusable.yml Co-authored-by: AL Berez --- .github/workflows/tests-integration-reusable.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/tests-integration-reusable.yml b/.github/workflows/tests-integration-reusable.yml index cf91d0beadc..36bdc768c9f 100644 --- a/.github/workflows/tests-integration-reusable.yml +++ b/.github/workflows/tests-integration-reusable.yml @@ -101,7 +101,6 @@ jobs: if: ${{ inputs.name == 'cats' }} run: | set -eu - ENV=$(jq -r .name metadata.json) API="$(jq -r .cf.api_url metadata.json)" DOMAIN=$(echo $API | sed "s/^api\.//")