File tree 1 file changed +15
-18
lines changed 1 file changed +15
-18
lines changed Original file line number Diff line number Diff line change 1
- """llama-cpp-python server from scratch in a single file.
1
+ """Example FastAPI server for llama.cpp.
2
+ To run this example:
3
+ ```bash
4
+ pip install fastapi uvicorn sse-starlette pydantic-settings
5
+ export MODEL=../models/7B/...
6
+ ```
7
+ Then run:
8
+ ```
9
+ uvicorn llama_cpp.server.app:create_app --reload
10
+ ```
11
+ or
12
+ ```
13
+ python3 -m llama_cpp.server
14
+ ```
15
+ Then visit http://localhost:8000/docs to see the interactive API docs.
2
16
"""
3
17
4
- # import llama_cpp
5
-
6
- # path = b"../../models/Qwen1.5-0.5B-Chat-GGUF/qwen1_5-0_5b-chat-q8_0.gguf"
7
-
8
- # model_params = llama_cpp.llama_model_default_params()
9
- # model = llama_cpp.llama_load_model_from_file(path, model_params)
10
-
11
- # if model is None:
12
- # raise RuntimeError(f"Failed to load model from file: {path}")
13
-
14
-
15
- # ctx_params = llama_cpp.llama_context_default_params()
16
- # ctx = llama_cpp.llama_new_context_with_model(model, ctx_params)
17
-
18
- # if ctx is None:
19
- # raise RuntimeError("Failed to create context")
20
-
21
18
22
19
from __future__ import annotations
23
20
You can’t perform that action at this time.
0 commit comments