Skip to content

Commit 9d36354

Browse files
authored
parse tokenizer_backend=None properly (#2509)
1 parent 867413f commit 9d36354

File tree

1 file changed

+4
-2
lines changed

1 file changed

+4
-2
lines changed

lm_eval/models/api_models.py

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ def __init__(
6262
# however the requests can be sent as a string if the API doesn't support token inputs.
6363
# use tokenized_requests=False
6464
tokenizer_backend: Optional[
65-
Literal["tiktoken", "huggingface", None]
65+
Literal["tiktoken", "huggingface", "None", "none"]
6666
] = "huggingface",
6767
truncate: bool = False,
6868
# number of concurrent requests. More useful if not batching
@@ -116,7 +116,9 @@ def __init__(
116116
"Concurrent requests are disabled. To enable concurrent requests, set `num_concurrent` > 1."
117117
)
118118
self._concurrent = int(num_concurrent)
119-
self.tokenizer_backend = tokenizer_backend
119+
self.tokenizer_backend = (
120+
None if tokenizer_backend in ("None", "none") else tokenizer_backend
121+
)
120122
self.add_bos_token = add_bos_token
121123
self.custom_prefix_token_id = custom_prefix_token_id
122124
self.tokenized_requests = tokenized_requests

0 commit comments

Comments
 (0)