[air] large tune/torch benchmark (ray-project#26763)

Co-authored-by: Kai Fricke <[email protected]>
Signed-off-by: Stefan van der Kleij <[email protected]>

release/air_tests/air_benchmarks/compute_gpu_8_g4_12xl.yaml

@@ -0,0 +1,15 @@
+cloud_id: {{env["ANYSCALE_CLOUD_ID"]}}
+region: us-west-2
+
+max_workers: 7
+
+head_node_type:
+    name: head_node
+    instance_type: g4dn.12xlarge
+
+worker_node_types:
+    - name: worker_node
+      instance_type: g4dn.12xlarge
+      max_workers: 7
+      min_workers: 7
+      use_spot: false

release/release_tests.yaml

@@ -414,6 +414,30 @@
 
   alert: default
 
+- name: air_benchmark_tune_torch_mnist_large_gpu
+  group: AIR tests
+  working_dir: air_tests/air_benchmarks
+
+  frequency: weekly
+  team: ml
+  env: staging
+
+  cluster:
+    cluster_env: app_config.yaml
+    cluster_compute: compute_gpu_8_g4_12xl.yaml
+
+  run:
+    timeout: 3600
+    script: python workloads/tune_torch_benchmark.py --num-runs 2 --num-trials 4 --num-workers 8 --use-gpu
+
+    wait_for_nodes:
+      num_nodes: 8
+
+    type: sdk_command
+    file_manager: job
+
+  alert: default
+
 
 # Ray AIR distributed Tensorflow benchmarks
 - name: air_benchmark_tensorflow_mnist_cpu_4x1