Add GCE variation for core release tests [3/n] (#34425)

- microbenchmark_38 - shuffle_20gb_with_state_api - object_store - many_actors - many_tasks - many_pgs - chaos_many_tasks_no_object_store - chaos_many_actors Signed-off-by: Jiajun Yao <[email protected]>
ray-project · Apr 15, 2023 · cb59e59 · cb59e59
1 parent bd8905b
commit cb59e59
Show file tree

Hide file tree

Showing 5 changed files with 167 additions and 0 deletions.
diff --git a/release/benchmarks/distributed_gce.yaml b/release/benchmarks/distributed_gce.yaml
@@ -0,0 +1,33 @@
+cloud_id: {{env["ANYSCALE_CLOUD_ID"]}}
+region: us-west1
+allowed_azs:
+    - us-west1-c
+
+max_workers: 999
+
+head_node_type:
+    name: head_node
+    instance_type: n2-standard-64 # Network optimized.
+    resources:
+      cpu: 0
+      custom_resources:
+        node: 1
+        small: 1
+
+worker_node_types:
+    - name: worker_node_m
+      instance_type: n2-standard-64
+      min_workers: 32
+      max_workers: 32
+      use_spot: false
+      resources:
+        custom_resources:
+          node: 1
+    - name: worker_node_r
+      instance_type: n2-standard-64
+      min_workers: 32
+      max_workers: 32
+      use_spot: false
+      resources:
+        custom_resources:
+          node: 1
diff --git a/release/benchmarks/object_store_gce.yaml b/release/benchmarks/object_store_gce.yaml
@@ -0,0 +1,23 @@
+cloud_id: {{env["ANYSCALE_CLOUD_ID"]}}
+region: us-west1
+allowed_azs:
+    - us-west1-c
+
+max_workers: 49
+
+head_node_type:
+    name: head_node
+    instance_type: n2-standard-64
+    resources:
+      custom_resources:
+        node: 1
+
+worker_node_types:
+    - name: worker_node
+      instance_type: n2-standard-8
+      min_workers: 49
+      max_workers: 49
+      use_spot: false
+      resources:
+        custom_resources:
+          node: 1
diff --git a/release/nightly_tests/chaos_test/compute_template_gce.yaml b/release/nightly_tests/chaos_test/compute_template_gce.yaml
@@ -0,0 +1,22 @@
+cloud_id: {{env["ANYSCALE_CLOUD_ID"]}}
+region: us-west1
+allowed_azs:
+    - us-west1-c
+
+head_node_type:
+    name: head_node
+    instance_type: n2-standard-64
+    resources:
+      cpu: 0
+      custom_resources:
+        head: 1
+
+worker_node_types:
+   - name: worker_node
+     instance_type: n2-standard-16
+     min_workers: 9
+     max_workers: 9
+     use_spot: true
+     resources:
+      custom_resources:
+        worker: 1
diff --git a/release/nightly_tests/shuffle/shuffle_compute_single_gce.yaml b/release/nightly_tests/shuffle/shuffle_compute_single_gce.yaml
@@ -0,0 +1,21 @@
+cloud_id: {{env["ANYSCALE_CLOUD_ID"]}}
+region: us-west1
+allowed_azs:
+    - us-west1-c
+
+max_workers: 0
+
+gcp_advanced_configurations_json:
+  instance_properties:
+    disks:
+      - boot: true
+        auto_delete: true
+        initialize_params:
+          disk_size_gb: 500
+
+head_node_type:
+    name: head_node
+    instance_type: n2-highmem-16
+    resources: {"object_store_memory": 21474836480}
+
+worker_node_types: []
diff --git a/release/release_tests.yaml b/release/release_tests.yaml
@@ -3836,6 +3836,16 @@
     timeout: 1800
     script: OMP_NUM_THREADS=64 RAY_ADDRESS=local python run_microbenchmark.py
 
+  variations:
+    - __suffix__: aws
+    - __suffix__: gce
+      env: gce
+      frequency: manual
+      cluster:
+        cluster_env: app_config.yaml
+        cluster_compute: tpl_64_gce.yaml
+
+
 - name: benchmark_worker_startup
   group: core-daily-test
   team: core
@@ -3959,6 +3969,14 @@
     script: python stress_tests/test_state_api_with_other_tests.py
       nightly_tests/shuffle/shuffle_test.py --test-args="--num-partitions=100 --partition-size=200e6"
 
+  variations:
+    - __suffix__: aws
+    - __suffix__: gce
+      env: gce
+      frequency: manual
+      cluster:
+        cluster_env: shuffle/shuffle_with_state_api_app_config.yaml
+        cluster_compute: shuffle/shuffle_compute_single_gce.yaml
 
 - name: stress_test_many_tasks
   group: core-daily-test
@@ -4254,6 +4272,14 @@
     wait_for_nodes:
       num_nodes: 50
 
+  variations:
+    - __suffix__: aws
+    - __suffix__: gce
+      env: gce
+      frequency: manual
+      cluster:
+        cluster_env: app_config.yaml
+        cluster_compute: object_store_gce.yaml
 
 - name: many_actors
   group: core-scalability-test
@@ -4271,6 +4297,14 @@
     wait_for_nodes:
       num_nodes: 65
 
+  variations:
+    - __suffix__: aws
+    - __suffix__: gce
+      env: gce
+      frequency: manual
+      cluster:
+        cluster_env: app_config.yaml
+        cluster_compute: distributed_gce.yaml
 
 - name: many_actors_smoke_test
   group: core-scalability-test
@@ -4305,6 +4339,14 @@
     wait_for_nodes:
       num_nodes: 65
 
+  variations:
+    - __suffix__: aws
+    - __suffix__: gce
+      env: gce
+      frequency: manual
+      cluster:
+        cluster_env: app_config.yaml
+        cluster_compute: distributed_gce.yaml
 
 - name: many_pgs
   group: core-scalability-test
@@ -4322,6 +4364,15 @@
     wait_for_nodes:
       num_nodes: 65
 
+  variations:
+    - __suffix__: aws
+    - __suffix__: gce
+      env: gce
+      frequency: manual
+      cluster:
+        cluster_env: app_config.yaml
+        cluster_compute: distributed_gce.yaml
+
 
 - name: many_pgs_smoke_test
   group: core-scalability-test
@@ -4831,6 +4882,14 @@
     prepare: python setup_chaos.py --no-start
     script: python chaos_test/test_chaos_basic.py --workload=tasks
 
+  variations:
+    - __suffix__: aws
+    - __suffix__: gce
+      env: gce
+      frequency: manual
+      cluster:
+        cluster_env: chaos_test/app_config.yaml
+        cluster_compute: chaos_test/compute_template_gce.yaml
 
 - name: chaos_many_actors
   group: core-nightly-test
@@ -4849,6 +4908,15 @@
     prepare: python setup_chaos.py --no-start
     script: python chaos_test/test_chaos_basic.py --workload=actors
 
+  variations:
+    - __suffix__: aws
+    - __suffix__: gce
+      env: gce
+      frequency: manual
+      cluster:
+        cluster_env: chaos_test/app_config.yaml
+        cluster_compute: chaos_test/compute_template_gce.yaml
+
 
 - name: chaos_dask_on_ray_large_scale_test_no_spilling
   group: data-tests