example-llama3instruct.env

MODEL_PATH="/absolute/path/to/model/llama3-instruct.gguf"

BOT="<|begin_of_text|>"
PREFIX_TEMPLATE="<|start_header_id|>{agent}<|end_header_id|>
"
EOS="<|eot_id|>"
AGENT_SYSTEM="system"
AGENT_USER="user"
AGENT_ASSISTANT="assistant"
SYSTEM_PROMPT="You are a helpful developer assistant, answer all the questions correctly and concisely."
ASSISTANT_INITIAL_MESSAGE="Hello, do you have any question?"

REAL_TIME=1
N_CTX=8000
N_GENERATE=2000
TEMPERATURE=0.8
TOP_P=0.9
TOP_K=40
SEED=69
USE_MMAP=1
USE_MLOCK=1
USE_GPU=1