File tree 1 file changed +4
-2
lines changed 1 file changed +4
-2
lines changed Original file line number Diff line number Diff line change @@ -62,7 +62,7 @@ def __init__(
62
62
# however the requests can be sent as a string if the API doesn't support token inputs.
63
63
# use tokenized_requests=False
64
64
tokenizer_backend : Optional [
65
- Literal ["tiktoken" , "huggingface" , None ]
65
+ Literal ["tiktoken" , "huggingface" , " None" , "none" ]
66
66
] = "huggingface" ,
67
67
truncate : bool = False ,
68
68
# number of concurrent requests. More useful if not batching
@@ -116,7 +116,9 @@ def __init__(
116
116
"Concurrent requests are disabled. To enable concurrent requests, set `num_concurrent` > 1."
117
117
)
118
118
self ._concurrent = int (num_concurrent )
119
- self .tokenizer_backend = tokenizer_backend
119
+ self .tokenizer_backend = (
120
+ None if tokenizer_backend in ("None" , "none" ) else tokenizer_backend
121
+ )
120
122
self .add_bos_token = add_bos_token
121
123
self .custom_prefix_token_id = custom_prefix_token_id
122
124
self .tokenized_requests = tokenized_requests
You can’t perform that action at this time.
0 commit comments