Skip to content

Commit 2fd41f9

Browse files
committed
Add moondream support (wip)
1 parent 3cef09c commit 2fd41f9

File tree

2 files changed

+40
-0
lines changed

2 files changed

+40
-0
lines changed

llama_cpp/llama_chat_format.py

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2477,6 +2477,32 @@ def from_pretrained(
24772477
**kwargs,
24782478
)
24792479

2480+
class MoondreamChatHanlder(Llava15ChatHandler):
2481+
# Chat Format:
2482+
# <image>\n\nQuestion: {prompt}\n\nAnswer:
2483+
CHAT_FORMAT = (
2484+
"{% for message in messages %}"
2485+
"{% if message.role == 'user' %}"
2486+
"{% if message.content is iterable %}"
2487+
"{% for content in message.content %}"
2488+
"{% if content.type == 'image_url' %}"
2489+
"{{ content.image_url }}"
2490+
"{% endif %}"
2491+
"{% if content.type == 'text' %}"
2492+
"Question: {{ content.text }}"
2493+
"{% endif %}"
2494+
"{% endfor %}"
2495+
"{% endif %}"
2496+
"{% endif %}"
2497+
"{% if message.role == 'assistant' %}"
2498+
"Answer: {{ message.content }}"
2499+
"{% endif %}"
2500+
"{% endfor %}"
2501+
"{% if add_generation_prompt %}"
2502+
"Answer: "
2503+
"{% endif %}"
2504+
)
2505+
24802506

24812507
@register_chat_completion_handler("chatml-function-calling")
24822508
def chatml_function_calling(

llama_cpp/server/model.py

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,20 @@ def load_llama_from_model_settings(settings: ModelSettings) -> llama_cpp.Llama:
8484
chat_handler = llama_cpp.llama_chat_format.Llava15ChatHandler(
8585
clip_model_path=settings.clip_model_path, verbose=settings.verbose
8686
)
87+
elif settings.chat_format == "moondream":
88+
assert settings.clip_model_path is not None, "clip model not found"
89+
if settings.hf_model_repo_id is not None:
90+
chat_handler = (
91+
llama_cpp.llama_chat_format.MoondreamChatHanlder.from_pretrained(
92+
repo_id=settings.hf_model_repo_id,
93+
filename=settings.clip_model_path,
94+
verbose=settings.verbose,
95+
)
96+
)
97+
else:
98+
chat_handler = llama_cpp.llama_chat_format.MoondreamChatHanlder(
99+
clip_model_path=settings.clip_model_path, verbose=settings.verbose
100+
)
87101
elif settings.chat_format == "hf-autotokenizer":
88102
assert (
89103
settings.hf_pretrained_model_name_or_path is not None

0 commit comments

Comments
 (0)