Skip to content

Commit 690aaa1

Browse files
authored
Update __main__.py
1 parent 75f6812 commit 690aaa1

File tree

1 file changed

+15
-18
lines changed

1 file changed

+15
-18
lines changed

llama_cpp/server/__main__.py

Lines changed: 15 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,20 @@
1-
"""llama-cpp-python server from scratch in a single file.
1+
"""Example FastAPI server for llama.cpp.
2+
To run this example:
3+
```bash
4+
pip install fastapi uvicorn sse-starlette pydantic-settings
5+
export MODEL=../models/7B/...
6+
```
7+
Then run:
8+
```
9+
uvicorn llama_cpp.server.app:create_app --reload
10+
```
11+
or
12+
```
13+
python3 -m llama_cpp.server
14+
```
15+
Then visit http://localhost:8000/docs to see the interactive API docs.
216
"""
317

4-
# import llama_cpp
5-
6-
# path = b"../../models/Qwen1.5-0.5B-Chat-GGUF/qwen1_5-0_5b-chat-q8_0.gguf"
7-
8-
# model_params = llama_cpp.llama_model_default_params()
9-
# model = llama_cpp.llama_load_model_from_file(path, model_params)
10-
11-
# if model is None:
12-
# raise RuntimeError(f"Failed to load model from file: {path}")
13-
14-
15-
# ctx_params = llama_cpp.llama_context_default_params()
16-
# ctx = llama_cpp.llama_new_context_with_model(model, ctx_params)
17-
18-
# if ctx is None:
19-
# raise RuntimeError("Failed to create context")
20-
2118

2219
from __future__ import annotations
2320

0 commit comments

Comments
 (0)