cuda: check for gpu instead of /dev/nvidiactl

The check for `/dev/nvidiactl` to determine if the CUDA plugin can be used is unreliable because in some cases the default path for driver installation is different [1]. This patch changes the logic to check if a GPU device is available in `/proc/driver/nvidia/gpus/`. This is a more accurate indicator, and the subsequent check for `--action` option would confirm if the NVIDIA driver supports checkpoint/restore. [1] https://github.com/NVIDIA/gpu-operator Fixes: #2509 Signed-off-by: Radostin Stoyanov <[email protected]>
checkpoint-restore · Nov 8, 2024 · d7860aa · d7860aa
1 parent 216d804
commit d7860aa
Show file tree

Hide file tree

Showing 2 changed files with 14 additions and 3 deletions.
diff --git a/plugins/cuda/cuda_plugin.c b/plugins/cuda/cuda_plugin.c
@@ -470,6 +470,17 @@ int cuda_plugin_resume_devices_late(int pid)
 }
 CR_PLUGIN_REGISTER_HOOK(CR_PLUGIN_HOOK__RESUME_DEVICES_LATE, cuda_plugin_resume_devices_late)
 
+static bool has_nvidia_gpu(void)
+{
+	const char *gpu_path = "/proc/driver/nvidia/gpus/";
+	struct stat sb;
+
+	if (stat(gpu_path, &sb) != 0)
+		return false;
+
+	return S_ISDIR(sb.st_mode);
+}
+
 int cuda_plugin_init(int stage)
 {
 	int ret;
@@ -481,8 +492,8 @@ int cuda_plugin_init(int stage)
 		}
 	}
 
-	if (!fault_injected(FI_PLUGIN_CUDA_FORCE_ENABLE) && access("/dev/nvidiactl", F_OK)) {
-		pr_info("/dev/nvidiactl doesn't exist. The CUDA plugin is disabled.\n");
+	if (!fault_injected(FI_PLUGIN_CUDA_FORCE_ENABLE) && !has_nvidia_gpu()) {
+		pr_info("No GPU device found; CUDA plugin is disabled\n");
 		plugin_disabled = true;
 		return 0;
 	}

diff --git a/scripts/ci/run-ci-tests.sh b/scripts/ci/run-ci-tests.sh
@@ -364,4 +364,4 @@ make -C plugins/amdgpu/ test_topology_remap
 ./test/zdtm.py run -t zdtm/static/maps00 -t zdtm/static/maps02 --criu-plugin amdgpu cuda
 ./test/zdtm.py run -t zdtm/static/busyloop00 --criu-plugin inventory_test_enabled inventory_test_disabled
 
-./test/zdtm.py run -t zdtm/static/sigpending -t zdtm/static/pthread00 --mocked-cuda-checkpoint --fault 138
+./test/zdtm.py run -t zdtm/static/sigpending -t zdtm/static/pthread00 --mocked-cuda-checkpoint --criu-plugin cuda --fault 138