Merge pull request #35 from greenplum-db/pipeline_fix

fix: pipeline down
greenplum-db · Mar 31, 2022 · 15d08cc · 15d08cc
2 parents 8d85719 + fb54c0e
commit 15d08cc
Show file tree

Hide file tree

Showing 2 changed files with 92 additions and 47 deletions.
diff --git a/concourse/pipeline-release.yml b/concourse/pipeline-release.yml
@@ -6,6 +6,7 @@ groups:
   - greenplumPython_release
 
 ccp_create_params_anchor: &ccp_default_params
+  plugin_dir: ../../terraform.d/plugin-cache/linux_amd64
   action: create
   delete_on_failure: true
   generate_random_name: true
@@ -28,12 +29,13 @@ ccp_gen_cluster_default_params_anchor: &ccp_gen_cluster_default_params
 ccp_destroy_anchor: &ccp_destroy
   put: terraform
   params:
+    plugin_dir: ../../terraform.d/plugin-cache/linux_amd64
     action: destroy
     env_name_file: terraform/name
     terraform_source: ccp_src/google/
     vars:
       aws_instance-node-instance_type: t2.micro #t2.micro is ignored in destroy, but aws_instance-node-instance_type is required.
-      aws_ebs_volume_type: standard
+      instance_type: n1-standard-1
   get_params:
     action: destroy
 
@@ -43,10 +45,9 @@ set_failed_anchor: &set_failed
     config:
       platform: linux
       image_resource:
-        type: docker-image
+        type: registry-image
         source:
-          repository: pivotaldata/ccp
-          tag: "7"
+          repository: gcr.io/data-gpdb-public-images/ccp
       inputs:
         - name: ccp_src
         - name: terraform
@@ -63,34 +64,31 @@ set_failed_anchor: &set_failed
         BUCKET_PATH: clusters-google/
         BUCKET_NAME: {{tf-bucket-name}}
 #####################################################
-
 resource_types:
-- name: gcs
-  type: docker-image
-  source:
-    repository: frodenas/gcs-resource
-
-- name: terraform
-  type: docker-image
-  source:
-    repository: ljfranklin/terraform-resource
-    tag: 0.11.14
+    - name: terraform
+      type: registry-image
+      source:
+        repository: ljfranklin/terraform-resource
+        tag: 0.11.14
+    - name: gcs
+      type: registry-image
+      source:
+        repository: frodenas/gcs-resource
 
 resources:
 # Image Resources
 - name: centos-gpdb-dev-7
-  type: docker-image
+  type: registry-image
   source:
-    repository: pivotaldata/centos-gpdb-dev
-    tag: '7-gcc6.2-llvm3.7'
+    repository: gcr.io/data-gpdb-public-images/gpdb5-centos7-build-test
+    tag: latest
 
 - name: ubuntu18-image-test
-  type: docker-image
+  type: registry-image
   source:
-    repository: pivotaldata/gpdb6-ubuntu18.04-test
+    repository: gcr.io/data-gpdb-public-images/gpdb6-ubuntu18.04-test
     tag: latest
 
-
 - name: GreenplumPython_src
   type: git
   source:
@@ -130,6 +128,15 @@ resources:
       bucket: {{tf-bucket-name}}
       bucket_path: clusters-google/
 
+- name: terraform.d
+  source:
+    access_key_id: ((bucket-access-key-id))
+    bucket: ccp-terraform-provider-plugins
+    region_name: ((aws-region))
+    secret_access_key: ((bucket-secret-access-key))
+    versioned_file: plugin-cache.tgz
+  type: s3
+
 - name: bin_gpdb_centos7
   type: gcs
   source:
@@ -191,6 +198,9 @@ jobs:
     - get: gpdb_binary
       resource: bin_gpdb_centos7
     - get: ccp_src
+    - get: terraform.d
+      params:
+        unpack: true
     - get: centos-gpdb-dev-7
     - get: plcontainer_pyclient_docker_image
       resource: plcontainer_docker_image_centos_python
@@ -205,6 +215,11 @@ jobs:
       vars:
         <<: *ccp_default_vars
         PLATFORM: centos7
+        instance_type: n1-standard-4
+        disk_size: 100
+        disk_type: pd-ssd
+        zone: {{google-zone}}
+        region: {{google-region}}
   - task: gen_cluster
     file: ccp_src/ci/tasks/gen_cluster.yml
     params:
@@ -233,6 +248,9 @@ jobs:
     - get: gpdb_binary
       resource: bin_gpdb_ubuntu18
     - get: ccp_src
+    - get: terraform.d
+      params:
+        unpack: true
     - get: ubuntu18-image-test
     - get: plcontainer_pyclient_docker_image
       resource: plcontainer_docker_image_centos_python
@@ -247,6 +265,11 @@ jobs:
       vars:
         <<: *ccp_default_vars
         PLATFORM: ubuntu18.04
+        instance_type: n1-standard-4
+        disk_size: 100
+        disk_type: pd-ssd
+        zone: {{google-zone}}
+        region: {{google-region}}
   - task: gen_cluster
     file: ccp_src/ci/tasks/gen_cluster.yml
     params:
@@ -278,4 +301,5 @@ jobs:
   - aggregate:
     - put: greenplumPython-release
       params:
-        file: greenplumPython-release/greenplum-python-*.tar.gz
+        file: greenplumPython-release/greenplum-python-*.tar.gz
+
diff --git a/concourse/pipeline.yml b/concourse/pipeline.yml
@@ -8,16 +8,15 @@ groups:
 #################### RESOURCES ######################
 #####################################################
 ccp_create_params_anchor: &ccp_default_params
+  plugin_dir: ../../terraform.d/plugin-cache/linux_amd64
   action: create
   delete_on_failure: true
   generate_random_name: true
   terraform_source: ccp_src/google/
 
 ccp_vars_anchor: &ccp_default_vars
-  instance_type: n1-standard-4
-  region: {{google-region}}
-  zone: {{google-zone}}
-  disk_size: 100
+  instance_type: n1-standard-2
+  number_of_nodes: 2
 
 ccp_gen_cluster_default_params_anchor: &ccp_gen_cluster_default_params
   AWS_ACCESS_KEY_ID: {{tf-machine-access-key-id}}
@@ -30,12 +29,13 @@ ccp_gen_cluster_default_params_anchor: &ccp_gen_cluster_default_params
 ccp_destroy_anchor: &ccp_destroy
   put: terraform
   params:
+    plugin_dir: ../../terraform.d/plugin-cache/linux_amd64
     action: destroy
     env_name_file: terraform/name
     terraform_source: ccp_src/google/
     vars:
       aws_instance-node-instance_type: t2.micro #t2.micro is ignored in destroy, but aws_instance-node-instance_type is required.
-      aws_ebs_volume_type: standard
+      instance_type: n1-standard-1
   get_params:
     action: destroy
 
@@ -45,10 +45,9 @@ set_failed_anchor: &set_failed
     config:
       platform: linux
       image_resource:
-        type: docker-image
+        type: registry-image
         source:
-          repository: pivotaldata/ccp
-          tag: "7"
+          repository: gcr.io/data-gpdb-public-images/ccp
       inputs:
         - name: ccp_src
         - name: terraform
@@ -65,34 +64,31 @@ set_failed_anchor: &set_failed
         BUCKET_PATH: clusters-google/
         BUCKET_NAME: {{tf-bucket-name}}
 #####################################################
-
 resource_types:
-- name: gcs
-  type: docker-image
-  source:
-    repository: frodenas/gcs-resource
-
-- name: terraform
-  type: docker-image
-  source:
-    repository: ljfranklin/terraform-resource
-    tag: 0.11.14
+    - name: terraform
+      type: registry-image
+      source:
+        repository: ljfranklin/terraform-resource
+        tag: 0.11.14
+    - name: gcs
+      type: registry-image
+      source:
+        repository: frodenas/gcs-resource
 
 resources:
 # Image Resources
 - name: centos-gpdb-dev-7
-  type: docker-image
+  type: registry-image
   source:
-    repository: pivotaldata/centos-gpdb-dev
-    tag: '7-gcc6.2-llvm3.7'
+    repository: gcr.io/data-gpdb-public-images/gpdb5-centos7-build-test
+    tag: latest
 
 - name: ubuntu18-image-test
-  type: docker-image
+  type: registry-image
   source:
-    repository: pivotaldata/gpdb6-ubuntu18.04-test
+    repository: gcr.io/data-gpdb-public-images/gpdb6-ubuntu18.04-test
     tag: latest
 
-
 - name: GreenplumPython_src
   type: git
   source:
@@ -132,6 +128,15 @@ resources:
       bucket: {{tf-bucket-name}}
       bucket_path: clusters-google/
 
+- name: terraform.d
+  source:
+    access_key_id: ((bucket-access-key-id))
+    bucket: ccp-terraform-provider-plugins
+    region_name: ((aws-region))
+    secret_access_key: ((bucket-secret-access-key))
+    versioned_file: plugin-cache.tgz
+  type: s3
+
 - name: bin_gpdb_centos7
   type: gcs
   source:
@@ -187,6 +192,9 @@ jobs:
     - get: gpdb_binary
       resource: bin_gpdb_centos7
     - get: ccp_src
+    - get: terraform.d
+      params:
+        unpack: true
     - get: centos-gpdb-dev-7
     - get: plcontainer_pyclient_docker_image
       resource: plcontainer_docker_image_centos_python
@@ -201,6 +209,11 @@ jobs:
       vars:
         <<: *ccp_default_vars
         PLATFORM: centos7
+        instance_type: n1-standard-4
+        disk_size: 100
+        disk_type: pd-ssd
+        zone: {{google-zone}}
+        region: {{google-region}}
   - task: gen_cluster
     file: ccp_src/ci/tasks/gen_cluster.yml
     params:
@@ -229,6 +242,9 @@ jobs:
     - get: gpdb_binary
       resource: bin_gpdb_ubuntu18
     - get: ccp_src
+    - get: terraform.d
+      params:
+        unpack: true
     - get: ubuntu18-image-test
     - get: plcontainer_pyclient_docker_image
       resource: plcontainer_docker_image_centos_python
@@ -243,6 +259,11 @@ jobs:
       vars:
         <<: *ccp_default_vars
         PLATFORM: ubuntu18.04
+        instance_type: n1-standard-4
+        disk_size: 100
+        disk_type: pd-ssd
+        zone: {{google-zone}}
+        region: {{google-region}}
   - task: gen_cluster
     file: ccp_src/ci/tasks/gen_cluster.yml
     params: