Skip to content

Commit

Permalink
env var for disabling kernels
Browse files Browse the repository at this point in the history
  • Loading branch information
LucasWilkinson committed Sep 10, 2024
1 parent abe41b6 commit 90f8bb6
Showing 1 changed file with 8 additions and 2 deletions.
10 changes: 8 additions & 2 deletions vllm/model_executor/layers/quantization/kernels/__init__.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import os
from typing import List, Optional, Type

from vllm.platforms import current_platform
Expand All @@ -24,6 +25,12 @@ def choose_mp_linear_kernel(

failure_reasons = []
for kernel in _POSSIBLE_KERNELS:
if kernel.__name__ in os.environ.get("VLLM_DISABLED_KERNELS", "")\
.split(","):
failure_reasons.append(
f' {kernel.__name__} disabled by environment variable')
continue

if kernel.get_min_capability() > compute_capability:
failure_reasons.append(
f"{kernel.__name__} requires capability "
Expand All @@ -35,8 +42,7 @@ def choose_mp_linear_kernel(
return kernel
else:
failure_reasons.append(
f' {kernel.__name__} cannot implement due to: {failure_reason}'
)
f' {kernel.__name__} cannot implement due to: {failure_reason}')

raise ValueError(
"Failed to find a kernel that can implement the "\
Expand Down

0 comments on commit 90f8bb6

Please sign in to comment.