Skip to content

Commit 12a7c6e

Browse files
committed
add model_is_mrope
1 parent d9ad251 commit 12a7c6e

File tree

1 file changed

+9
-0
lines changed

1 file changed

+9
-0
lines changed

vllm/worker/cpu_model_runner.py

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -422,6 +422,15 @@ def __init__(
422422
raise NotImplementedError(
423423
STR_NOT_IMPL_ENC_DEC_ERR_STRS['STR_NOT_IMPL_ENC_DEC_CPU'])
424424

425+
@property
426+
def model_is_mrope(self) -> bool:
427+
"""Detect if the model has "mrope" rope_scaling type.
428+
mrope requires keep "rope_deltas" between prompt and decoding phases."""
429+
rope_scaling = getattr(self.model_config.hf_config, "rope_scaling", {})
430+
if rope_scaling is None:
431+
return False
432+
return rope_scaling.get("type", None) == "mrope"
433+
425434
def load_model(self) -> None:
426435
self.model = get_model(model_config=self.model_config,
427436
load_config=self.load_config,

0 commit comments

Comments
 (0)