From 1d6d76330e05182c7b419499ae33a82903a65732 Mon Sep 17 00:00:00 2001 From: Tyler Osterberg Date: Tue, 10 Sep 2024 17:16:23 -0700 Subject: [PATCH] [python] Update vllm rolling batcher sampling params for 0.6.0 support (#2376) --- .../setup/djl_python/rolling_batch/vllm_rolling_batch.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/engines/python/setup/djl_python/rolling_batch/vllm_rolling_batch.py b/engines/python/setup/djl_python/rolling_batch/vllm_rolling_batch.py index 4b216a793..3f3d665e2 100644 --- a/engines/python/setup/djl_python/rolling_batch/vllm_rolling_batch.py +++ b/engines/python/setup/djl_python/rolling_batch/vllm_rolling_batch.py @@ -23,7 +23,10 @@ from djl_python.properties_manager.vllm_rb_properties import VllmRbProperties from typing import List -VLLM_GENERATION_PARAMS = set(SamplingParams().__dict__.keys()) +# FIXME: Once all vllm versions are past 0.6.0 we can move to just struct_fields +VLLM_GENERATION_PARAMS = set(SamplingParams().__struct_fields__) if hasattr( + SamplingParams(), "__struct_fields__") else set( + SamplingParams().__dict__.keys()) class VLLMRollingBatch(RollingBatch):