From fb54c0e858c8d1c049f77a8963520b5d7667c2ab Mon Sep 17 00:00:00 2001 From: yihong0618 Date: Wed, 30 Mar 2022 15:54:03 +0800 Subject: [PATCH] fix: pipeline down reasons: 1. url link change from pivotal -> gcr.io 2. now we need to use terraform.d 3. some ccp env changed --- concourse/pipeline-release.yml | 68 +++++++++++++++++++++----------- concourse/pipeline.yml | 71 ++++++++++++++++++++++------------ 2 files changed, 92 insertions(+), 47 deletions(-) diff --git a/concourse/pipeline-release.yml b/concourse/pipeline-release.yml index 3cf69b05..218341ba 100644 --- a/concourse/pipeline-release.yml +++ b/concourse/pipeline-release.yml @@ -6,6 +6,7 @@ groups: - greenplumPython_release ccp_create_params_anchor: &ccp_default_params + plugin_dir: ../../terraform.d/plugin-cache/linux_amd64 action: create delete_on_failure: true generate_random_name: true @@ -28,12 +29,13 @@ ccp_gen_cluster_default_params_anchor: &ccp_gen_cluster_default_params ccp_destroy_anchor: &ccp_destroy put: terraform params: + plugin_dir: ../../terraform.d/plugin-cache/linux_amd64 action: destroy env_name_file: terraform/name terraform_source: ccp_src/google/ vars: aws_instance-node-instance_type: t2.micro #t2.micro is ignored in destroy, but aws_instance-node-instance_type is required. - aws_ebs_volume_type: standard + instance_type: n1-standard-1 get_params: action: destroy @@ -43,10 +45,9 @@ set_failed_anchor: &set_failed config: platform: linux image_resource: - type: docker-image + type: registry-image source: - repository: pivotaldata/ccp - tag: "7" + repository: gcr.io/data-gpdb-public-images/ccp inputs: - name: ccp_src - name: terraform @@ -63,34 +64,31 @@ set_failed_anchor: &set_failed BUCKET_PATH: clusters-google/ BUCKET_NAME: {{tf-bucket-name}} ##################################################### - resource_types: -- name: gcs - type: docker-image - source: - repository: frodenas/gcs-resource - -- name: terraform - type: docker-image - source: - repository: ljfranklin/terraform-resource - tag: 0.11.14 + - name: terraform + type: registry-image + source: + repository: ljfranklin/terraform-resource + tag: 0.11.14 + - name: gcs + type: registry-image + source: + repository: frodenas/gcs-resource resources: # Image Resources - name: centos-gpdb-dev-7 - type: docker-image + type: registry-image source: - repository: pivotaldata/centos-gpdb-dev - tag: '7-gcc6.2-llvm3.7' + repository: gcr.io/data-gpdb-public-images/gpdb5-centos7-build-test + tag: latest - name: ubuntu18-image-test - type: docker-image + type: registry-image source: - repository: pivotaldata/gpdb6-ubuntu18.04-test + repository: gcr.io/data-gpdb-public-images/gpdb6-ubuntu18.04-test tag: latest - - name: GreenplumPython_src type: git source: @@ -130,6 +128,15 @@ resources: bucket: {{tf-bucket-name}} bucket_path: clusters-google/ +- name: terraform.d + source: + access_key_id: ((bucket-access-key-id)) + bucket: ccp-terraform-provider-plugins + region_name: ((aws-region)) + secret_access_key: ((bucket-secret-access-key)) + versioned_file: plugin-cache.tgz + type: s3 + - name: bin_gpdb_centos7 type: gcs source: @@ -191,6 +198,9 @@ jobs: - get: gpdb_binary resource: bin_gpdb_centos7 - get: ccp_src + - get: terraform.d + params: + unpack: true - get: centos-gpdb-dev-7 - get: plcontainer_pyclient_docker_image resource: plcontainer_docker_image_centos_python @@ -205,6 +215,11 @@ jobs: vars: <<: *ccp_default_vars PLATFORM: centos7 + instance_type: n1-standard-4 + disk_size: 100 + disk_type: pd-ssd + zone: {{google-zone}} + region: {{google-region}} - task: gen_cluster file: ccp_src/ci/tasks/gen_cluster.yml params: @@ -233,6 +248,9 @@ jobs: - get: gpdb_binary resource: bin_gpdb_ubuntu18 - get: ccp_src + - get: terraform.d + params: + unpack: true - get: ubuntu18-image-test - get: plcontainer_pyclient_docker_image resource: plcontainer_docker_image_centos_python @@ -247,6 +265,11 @@ jobs: vars: <<: *ccp_default_vars PLATFORM: ubuntu18.04 + instance_type: n1-standard-4 + disk_size: 100 + disk_type: pd-ssd + zone: {{google-zone}} + region: {{google-region}} - task: gen_cluster file: ccp_src/ci/tasks/gen_cluster.yml params: @@ -278,4 +301,5 @@ jobs: - aggregate: - put: greenplumPython-release params: - file: greenplumPython-release/greenplum-python-*.tar.gz \ No newline at end of file + file: greenplumPython-release/greenplum-python-*.tar.gz + diff --git a/concourse/pipeline.yml b/concourse/pipeline.yml index 1a675237..deb3052b 100644 --- a/concourse/pipeline.yml +++ b/concourse/pipeline.yml @@ -8,16 +8,15 @@ groups: #################### RESOURCES ###################### ##################################################### ccp_create_params_anchor: &ccp_default_params + plugin_dir: ../../terraform.d/plugin-cache/linux_amd64 action: create delete_on_failure: true generate_random_name: true terraform_source: ccp_src/google/ ccp_vars_anchor: &ccp_default_vars - instance_type: n1-standard-4 - region: {{google-region}} - zone: {{google-zone}} - disk_size: 100 + instance_type: n1-standard-2 + number_of_nodes: 2 ccp_gen_cluster_default_params_anchor: &ccp_gen_cluster_default_params AWS_ACCESS_KEY_ID: {{tf-machine-access-key-id}} @@ -30,12 +29,13 @@ ccp_gen_cluster_default_params_anchor: &ccp_gen_cluster_default_params ccp_destroy_anchor: &ccp_destroy put: terraform params: + plugin_dir: ../../terraform.d/plugin-cache/linux_amd64 action: destroy env_name_file: terraform/name terraform_source: ccp_src/google/ vars: aws_instance-node-instance_type: t2.micro #t2.micro is ignored in destroy, but aws_instance-node-instance_type is required. - aws_ebs_volume_type: standard + instance_type: n1-standard-1 get_params: action: destroy @@ -45,10 +45,9 @@ set_failed_anchor: &set_failed config: platform: linux image_resource: - type: docker-image + type: registry-image source: - repository: pivotaldata/ccp - tag: "7" + repository: gcr.io/data-gpdb-public-images/ccp inputs: - name: ccp_src - name: terraform @@ -65,34 +64,31 @@ set_failed_anchor: &set_failed BUCKET_PATH: clusters-google/ BUCKET_NAME: {{tf-bucket-name}} ##################################################### - resource_types: -- name: gcs - type: docker-image - source: - repository: frodenas/gcs-resource - -- name: terraform - type: docker-image - source: - repository: ljfranklin/terraform-resource - tag: 0.11.14 + - name: terraform + type: registry-image + source: + repository: ljfranklin/terraform-resource + tag: 0.11.14 + - name: gcs + type: registry-image + source: + repository: frodenas/gcs-resource resources: # Image Resources - name: centos-gpdb-dev-7 - type: docker-image + type: registry-image source: - repository: pivotaldata/centos-gpdb-dev - tag: '7-gcc6.2-llvm3.7' + repository: gcr.io/data-gpdb-public-images/gpdb5-centos7-build-test + tag: latest - name: ubuntu18-image-test - type: docker-image + type: registry-image source: - repository: pivotaldata/gpdb6-ubuntu18.04-test + repository: gcr.io/data-gpdb-public-images/gpdb6-ubuntu18.04-test tag: latest - - name: GreenplumPython_src type: git source: @@ -132,6 +128,15 @@ resources: bucket: {{tf-bucket-name}} bucket_path: clusters-google/ +- name: terraform.d + source: + access_key_id: ((bucket-access-key-id)) + bucket: ccp-terraform-provider-plugins + region_name: ((aws-region)) + secret_access_key: ((bucket-secret-access-key)) + versioned_file: plugin-cache.tgz + type: s3 + - name: bin_gpdb_centos7 type: gcs source: @@ -187,6 +192,9 @@ jobs: - get: gpdb_binary resource: bin_gpdb_centos7 - get: ccp_src + - get: terraform.d + params: + unpack: true - get: centos-gpdb-dev-7 - get: plcontainer_pyclient_docker_image resource: plcontainer_docker_image_centos_python @@ -201,6 +209,11 @@ jobs: vars: <<: *ccp_default_vars PLATFORM: centos7 + instance_type: n1-standard-4 + disk_size: 100 + disk_type: pd-ssd + zone: {{google-zone}} + region: {{google-region}} - task: gen_cluster file: ccp_src/ci/tasks/gen_cluster.yml params: @@ -229,6 +242,9 @@ jobs: - get: gpdb_binary resource: bin_gpdb_ubuntu18 - get: ccp_src + - get: terraform.d + params: + unpack: true - get: ubuntu18-image-test - get: plcontainer_pyclient_docker_image resource: plcontainer_docker_image_centos_python @@ -243,6 +259,11 @@ jobs: vars: <<: *ccp_default_vars PLATFORM: ubuntu18.04 + instance_type: n1-standard-4 + disk_size: 100 + disk_type: pd-ssd + zone: {{google-zone}} + region: {{google-region}} - task: gen_cluster file: ccp_src/ci/tasks/gen_cluster.yml params: