low_level_api_chat_cpp.py

"""
This is an example implementation of main.cpp from llama.cpp
Quirks:
 * Its not exactly alike since this port is designed around programmatic I/O
 * Input is always echoed if on, so it should be turned off when using "input()"
 * The first antiprompt should be the userprompt like "\nUser:", 
   because its added when n_predict is reached (aka generation ended prematurely)
 * n_predict can be set to -1 for unlimited length responses (or just a really high value)
 * Instruction mode adds its own antiprompt.
   You should also still be feeding the model with a "primer" prompt that 
   shows it the expected format.
"""

import ctypes
import sys
from time import time
from os import cpu_count, path

import llama_cpp
from common import GptParams, gpt_params_parse, gpt_random_prompt
import util


# A LLaMA interactive session
class LLaMAInteract:
    def __init__(self, params: GptParams) -> None:
        # input args
        self.params = params
        if self.params.path_session is None:
            self.params.path_session = ""
        if self.params.antiprompt is None:
            self.params.antiprompt = ""

        if self.params.perplexity:
            raise NotImplementedError(
                """************
please use the 'perplexity' tool for perplexity calculations
************"""
            )

        if self.params.embedding:
            raise NotImplementedError(
                """************
please use the 'embedding' tool for embedding calculations
************"""
            )

        if self.params.n_ctx > 2048:
            print(
                f"""warning: model does not support \
context sizes greater than 2048 tokens ({self.params.n_ctx} \
specified) expect poor results""",
                file=sys.stderr,
            )

        if self.params.seed <= 0:
            self.params.seed = int(time())

        print(f"seed = {self.params.seed}", file=sys.stderr)

        if self.params.random_prompt:
            self.params.prompt = gpt_random_prompt(self.params.seed)

        # runtime args
        self.input_consumed = 0
        self.n_past = 0
        self.n_session_consumed = 0
        self.first_antiprompt = []
        self.remaining_tokens = self.params.n_predict
        self.output_echo = self.params.input_echo
        self.multibyte_fix = []

        # model load
        self.lparams = llama_cpp.llama_model_default_params()
        self.lparams.n_ctx = self.params.n_ctx
        self.lparams.n_parts = self.params.n_parts
        self.lparams.seed = self.params.seed
        self.lparams.memory_f16 = self.params.memory_f16
        self.lparams.use_mlock = self.params.use_mlock
        self.lparams.use_mmap = self.params.use_mmap

        self.model = llama_cpp.llama_load_model_from_file(
            self.params.model.encode("utf8"), self.lparams
        )

        # Context Params.
        self.cparams = llama_cpp.llama_context_default_params()

        self.ctx = llama_cpp.llama_new_context_with_model(self.model, self.cparams)
        if not self.ctx:
            raise RuntimeError(f"error: failed to load model '{self.params.model}'")

        if self.params.ignore_eos:
            self.params.logit_bias[llama_cpp.llama_token_eos()] = -float("inf")

        if len(self.params.lora_adapter) > 0:
            if (
                llama_cpp.llama_apply_lora_from_file(
                    self.ctx,
                    self.params.lora_adapter.encode("utf8"),
                    (
                        self.params.lora_base.encode("utf8")
                        if len(self.params.lora_base) > 0
                        else None
                    ),
                    self.params.n_threads,
                )
                != 0
            ):
                print("error: failed to apply lora adapter")
                return

        print(file=sys.stderr)
        print(
            f"system_info: n_threads = {self.params.n_threads} / {cpu_count()} \
| {llama_cpp.llama_print_system_info().decode('utf8')}",
            file=sys.stderr,
        )

        # determine the required inference memory per token:
        if self.params.mem_test:
            tmp = [0, 1, 2, 3]
            llama_cpp.llama_eval(
                self.ctx,
                (llama_cpp.c_int * len(tmp))(*tmp),
                len(tmp),
                0,
                self.n_threads,
            )
            llama_cpp.llama_print_timings(self.ctx)
            self.exit()
            return

        # create internal context
        self.n_ctx = llama_cpp.llama_n_ctx(self.ctx)

        # Add a space in front of the first character to match OG llama tokenizer behavior
        self.params.prompt = " " + self.params.prompt

        # Load prompt file
        if self.params.file:
            with open(self.params.file) as f:
                self.params.prompt = f.read()

        self.session_tokens: list[llama_cpp.llama_token] = []
        if len(self.params.path_session) > 0:
            print(
                f"attempting to load saved session from '{self.params.path_session}'",
                file=sys.stderr,
            )

            if path.exists(self.params.path_session):
                _session_tokens = (llama_cpp.llama_token * (self.params.n_ctx))()
                _n_token_count_out = llama_cpp.c_size_t()
                if (
                    llama_cpp.llama_load_session_file(
                        self.ctx,
                        self.params.path_session.encode("utf8"),
                        _session_tokens,
                        self.params.n_ctx,
                        ctypes.byref(_n_token_count_out),
                    )
                    != 1
                ):
                    print(
                        f"error: failed to load session file '{self.params.path_session}'",
                        file=sys.stderr,
                    )
                    return
                _n_token_count_out = _n_token_count_out.value
                self.session_tokens = _session_tokens[:_n_token_count_out]
                print(
                    f"loaded a session with prompt size of {_n_token_count_out} tokens",
                    file=sys.stderr,
                )
            else:
                print(f"session file does not exist, will create", file=sys.stderr)

        # tokenize the prompt
        self.embd = []
        self.embd_inp = self._tokenize(self.params.prompt)

        if len(self.embd_inp) > self.n_ctx - 4:
            raise RuntimeError(
                f"error: prompt is too long ({len(self.embd_inp)} tokens, max {self.params.n_ctx - 4})"
            )

        # debug message about similarity of saved session, if applicable
        self.n_matching_session_tokens = 0
        if len(self.session_tokens) > 0:
            for id in self.session_tokens:
                if (
                    self.n_matching_session_tokens >= len(self.embd_inp)
                    or id != self.embd_inp[self.n_matching_session_tokens]
                ):
                    break
                self.n_matching_session_tokens += 1

            if self.n_matching_session_tokens >= len(self.embd_inp):
                print(f"session file has exact match for prompt!")
            elif self.n_matching_session_tokens < (len(self.embd_inp) / 2):
                print(
                    f"warning: session file has low similarity to prompt ({self.n_matching_session_tokens} / {len(self.embd_inp)} tokens); will mostly be reevaluated"
                )
            else:
                print(
                    f"session file matches {self.n_matching_session_tokens} / {len(self.embd_inp)} tokens of prompt"
                )

        self.need_to_save_session = len(
            self.params.path_session
        ) > 0 and self.n_matching_session_tokens < (len(self.embd_inp) * 3 / 4)

        # number of tokens to keep when resetting context
        if (
            self.params.n_keep < 0
            or self.params.n_keep > len(self.embd_inp)
            or self.params.instruct
        ):
            self.params.n_keep = len(self.embd_inp)

        self.inp_prefix = self._tokenize(self.params.instruct_inp_prefix)
        self.inp_suffix = self._tokenize(self.params.instruct_inp_suffix, False)

        # in instruct mode, we inject a prefix and a suffix to each input by the user
        self.antiecho = None
        if self.params.instruct:
            self.params.interactive_start = True
            _ptn = self._tokenize(self.params.instruct_inp_prefix.strip(), False)
            self.first_antiprompt.append(_ptn)
            self.antiecho = util.IterSearch(_ptn)

        # enable interactive mode if reverse prompt or interactive start is specified
        if len(self.params.antiprompt) != 0 or self.params.interactive_start:
            self.params.interactive = True

        # determine newline token
        self.llama_token_newline = self._tokenize("\n", False)
        self.llama_token_eot = self._tokenize(" [end of text]\n", False)

        if self.params.verbose_prompt:
            print(
                f"""
prompt: '{self.params.prompt}'
number of tokens in prompt = {len(self.embd_inp)}""",
                file=sys.stderr,
            )

            for i in range(len(self.embd_inp)):
                print(
                    f"{self.embd_inp[i]} -> '{self.token_to_str(self.embd_inp[i])}'",
                    file=sys.stderr,
                )

            if self.params.n_keep > 0:
                print("static prompt based on n_keep: '")
                for i in range(self.params.n_keep):
                    print(self.token_to_str(self.embd_inp[i]), file=sys.stderr)
                print("'", file=sys.stderr)
            print(file=sys.stderr)

        if self.params.interactive:
            print("interactive mode on.", file=sys.stderr)

            if len(self.params.antiprompt) > 0:
                for antiprompt in self.params.antiprompt:
                    print(f"Reverse prompt: '{antiprompt}'", file=sys.stderr)

            if len(self.params.input_prefix) > 0:
                print(f"Input prefix: '{self.params.input_prefix}'", file=sys.stderr)

        print(
            f"""sampling: repeat_last_n = {self.params.repeat_last_n},\
repeat_penalty = {self.params.repeat_penalty},\
presence_penalty = {self.params.presence_penalty},\
frequency_penalty = {self.params.frequency_penalty},\
top_k = {self.params.top_k},\
top_n_sigma  = {self.params.top_n_sigma},\
top_p = {self.params.top_p},\
typical_p = {self.params.typical_p},\
temp = {self.params.temp},\
mirostat = {self.params.mirostat},\
mirostat_lr = {self.params.mirostat_eta},\
mirostat_ent = {self.params.mirostat_tau},\

xtc_threshold = {self.params.xtc_threshold},\
xtc_probability = {self.params.xtc_probability},\

dry_multiplier = {self.params.dry_multiplier},\
dry_base = {self.params.dry_base},\
dry_allowed_length = {self.params.dry_allowed_length},\
dry_penalty_last_n = {self.params.dry_penalty_last_n},\
dry_seq_breakers = {self.params.dry_seq_breakers},\

generate: n_ctx = {self.n_ctx},\
n_batch = {self.params.n_batch},\
n_predict = {self.params.n_predict},\
n_keep = {self.params.n_keep}

""",
            file=sys.stderr,
        )

        # determine antiprompt tokens
        for i in self.params.antiprompt:
            self.first_antiprompt.append(self._tokenize(i, False))

        self.last_n_tokens = [0] * self.n_ctx  # TODO: deque doesnt support slices

        if params.interactive:
            print(
                """== Running in interactive mode. ==
 - Press Ctrl+C to interject at any time.
 - Press Return to return control to LLaMa.
 - If you want to submit another line, end your input in '\\'.

""",
                file=sys.stderr,
            )
        self.set_color(util.CONSOLE_COLOR_PROMPT)

    # tokenize a prompt
    def _tokenize(self, prompt, bos=True):
        _arr = (llama_cpp.llama_token * ((len(prompt) + 1) * 4))()
        _n = llama_cpp.llama_tokenize(
            self.model,
            prompt.encode("utf8", errors="ignore"),
            len(prompt),
            _arr,
            len(_arr),
            bos,
            False,
        )
        return _arr[:_n]

    def set_color(self, c):
        if self.params.use_color:
            print(c, end="")

    def use_antiprompt(self):
        return len(self.first_antiprompt) > 0

    # generate tokens
    def generate(self):
        while (
            self.remaining_tokens > 0
            or self.params.interactive
            or self.params.n_predict == -1
        ):
            # predict
            if len(self.embd) > 0:
                # infinite text generation via context swapping
                # if we run out of context:
                # - take the n_keep first tokens from the original prompt (via n_past)
                # - take half of the last (n_ctx - n_keep) tokens and recompute the logits in a batch
                if self.n_past + len(self.embd) > self.n_ctx:
                    n_left = self.n_past - self.params.n_keep
                    self.n_past = self.params.n_keep

                    # insert n_left/2 tokens at the start of embd from last_n_tokens
                    _insert = self.last_n_tokens[
                        self.n_ctx - int(n_left / 2) - len(self.embd) : -len(self.embd)
                    ]
                    self.embd = _insert + self.embd
                    self.params.path_session = ""

                # try to reuse a matching prefix from the loaded session instead of re-eval (via n_past)
                if self.n_session_consumed < len(self.session_tokens):
                    for i in range(len(self.embd)):
                        if self.embd[i] != self.session_tokens[self.n_session_consumed]:
                            self.session_tokens = self.session_tokens[
                                : self.n_session_consumed
                            ]
                            break

                        self.n_past += 1
                        self.n_session_consumed += 1

                        if self.n_session_consumed >= len(self.session_tokens):
                            i += 1
                            break

                    if i > 0:
                        self.embd = self.embd[i:]

                # evaluate tokens in batches
                # embd is typically prepared beforehand to fit within a batch, but not always
                # TODO BUG: The batching code causes nonsensical generation
                """for i in range(0, len(self.embd), self.params.n_batch):
					n_eval = self.params.n_batch
					_arr = (llama_cpp.llama_token * n_eval)(*self.embd[i:i + n_eval])
					if llama_cpp.llama_eval(self.ctx, _arr, n_eval, self.n_past, self.params.n_threads) != 0:
						print(f"failed to eval")
						return
					
					self.n_past += n_eval"""

                if (
                    llama_cpp.llama_eval(
                        self.ctx,
                        (llama_cpp.llama_token * len(self.embd))(*self.embd),
                        len(self.embd),
                        self.n_past,
                    )
                    != 0
                ):
                    raise Exception("Failed to llama_eval!")

                if len(self.embd) > 0 and len(self.params.path_session) > 0:
                    self.session_tokens.extend(self.embd)
                    self.n_session_consumed = len(self.session_tokens)

            self.n_past += len(self.embd)
            self.embd = []
            if len(self.embd_inp) <= self.input_consumed:  # && !is_interacting
                # out of user input, sample next token
                top_k = (
                    llama_cpp.llama_n_vocab(self.ctx)
                    if self.params.top_k <= 0
                    else self.params.top_k
                )
                repeat_last_n = (
                    self.n_ctx
                    if self.params.repeat_last_n < 0
                    else self.params.repeat_last_n
                )

                # optionally save the session on first sample (for faster prompt loading next time)
                if len(self.params.path_session) > 0 and self.need_to_save_session:
                    self.need_to_save_session = False
                    llama_cpp.llama_save_session_file(
                        self.ctx,
                        self.params.path_session.encode("utf8"),
                        (llama_cpp.llama_token * len(self.session_tokens))(
                            *self.session_tokens
                        ),
                        len(self.session_tokens),
                    )

                id = 0

                logits = llama_cpp.llama_get_logits(self.ctx)
                n_vocab = llama_cpp.llama_n_vocab(self.model)

                # Apply params.logit_bias map
                for key, value in self.params.logit_bias.items():
                    logits[key] += value

                _arr = (llama_cpp.llama_token_data * n_vocab)(
                    *[
                        llama_cpp.llama_token_data(token_id, logits[token_id], 0.0)
                        for token_id in range(n_vocab)
                    ]
                )
                candidates_p = llama_cpp.ctypes.pointer(
                    llama_cpp.llama_token_data_array(_arr, len(_arr), False)
                )

                # Apply penalties
                nl_logit = logits[llama_cpp.llama_token_nl(self.ctx)]
                last_n_repeat = min(len(self.last_n_tokens), repeat_last_n, self.n_ctx)

                _arr = (llama_cpp.llama_token * last_n_repeat)(
                    *self.last_n_tokens[len(self.last_n_tokens) - last_n_repeat :]
                )
                llama_cpp.llama_sampler_init_penalties(
                    ctx=self.ctx,
                    candidates=candidates_p,
                    last_tokens_data=_arr,
                    penalty_last_n=last_n_repeat,
                    penalty_repeat=llama_cpp.c_float(self.params.repeat_penalty),
                    penalty_freq=llama_cpp.c_float(self.params.frequency_penalty),
                    penalty_present=llama_cpp.c_float(self.params.presence_penalty),
                )

                # NOT PRESENT IN CURRENT VERSION ?
                # llama_cpp.llama_sample_frequency_and_presence_penalti(self.ctx, candidates_p,
                # 	_arr,
                # 	last_n_repeat, llama_cpp.c_float(self.params.frequency_penalty), llama_cpp.c_float(self.params.presence_penalty))

                if not self.params.penalize_nl:
                    logits[llama_cpp.llama_token_nl()] = nl_logit

                if self.params.temp <= 0:
                    # Greedy sampling
                    id = llama_cpp.llama_sampler_init_greedy(self.ctx, candidates_p)
                else:
                    if self.params.mirostat == 1:
                        mirostat_mu = 2.0 * self.params.mirostat_tau
                        mirostat_m = 100
                        llama_cpp.llama_sampler_init_temp(
                            self.ctx, candidates_p, llama_cpp.c_float(self.params.temp)
                        )
                        id = llama_cpp.llama_sampler_init_mirostat(
                            self.ctx,
                            candidates_p,
                            llama_cpp.c_float(self.params.mirostat_tau),
                            llama_cpp.c_float(self.params.mirostat_eta),
                            llama_cpp.c_int(mirostat_m),
                            llama_cpp.c_float(mirostat_mu),
                        )
                    elif self.params.mirostat == 2:
                        mirostat_mu = 2.0 * self.params.mirostat_tau
                        llama_cpp.llama_sample_temperature(
                            self.ctx, candidates_p, llama_cpp.c_float(self.params.temp)
                        )
                        id = llama_cpp.llama_sampler_init_mirostat_v2(
                            self.ctx,
                            candidates_p,
                            llama_cpp.c_float(self.params.mirostat_tau),
                            llama_cpp.c_float(self.params.mirostat_eta),
                            llama_cpp.c_float(mirostat_mu),
                        )
                    else:
                        # Temperature sampling
                        llama_cpp.llama_sampler_init_top_k(
                            self.ctx,
                            candidates_p,
                            top_k,
                            min_keep=llama_cpp.c_size_t(1),
                        )
                        llama_cpp.llama_sampler_init_top_n_sigma(
                            self.ctx,
                            candidates_p,
                            llama_cpp.c_float(self.params.top_n_sigma),
                            min_keep=llama_cpp.c_size_t(1),
                        )
                        llama_cpp.llama_sampler_init_typical(
                            self.ctx,
                            candidates_p,
                            llama_cpp.c_float(self.params.typical_p),
                            min_keep=llama_cpp.c_size_t(1),
                        )
                        llama_cpp.llama_sampler_init_top_p(
                            self.ctx,
                            candidates_p,
                            llama_cpp.c_float(self.params.top_p),
                            min_keep=llama_cpp.c_size_t(1),
                        )
                        llama_cpp.llama_sampler_init_temp(
                            self.ctx, candidates_p, llama_cpp.c_float(self.params.temp)
                        )
                        id = llama_cpp.llama_sample_token(self.ctx, candidates_p)
                # print("`{}`".format(candidates_p.size))

                self.last_n_tokens.pop(0)
                self.last_n_tokens.append(id)

                # replace end of text token with newline token when in interactive mode
                if (
                    id == llama_cpp.llama_token_eos(self.ctx)
                    and self.params.interactive
                    and not self.params.instruct
                ):
                    id = self.llama_token_newline[0]
                    self.embd.append(id)
                    if self.use_antiprompt():
                        # tokenize and inject first reverse prompt
                        self.embd_inp += self.first_antiprompt[0]
                        for id in self.first_antiprompt[0]:
                            self.embd.append(id)
                else:
                    # add it to the context
                    self.embd.append(id)

                # echo this to console
                self.output_echo = True

                # decrement remaining sampling budget
                self.remaining_tokens -= 1
            else:
                # output to console if input echo is on
                self.output_echo = self.params.input_echo

                # some user input remains from prompt or interaction, forward it to processing
                while len(self.embd_inp) > self.input_consumed:
                    self.embd.append(self.embd_inp[self.input_consumed])
                    self.last_n_tokens.pop(0)
                    self.last_n_tokens.append(self.embd_inp[self.input_consumed])
                    self.input_consumed += 1
                    if len(self.embd) >= self.params.n_batch:
                        break

            # display tokens
            if self.output_echo:
                for id in self.embd:
                    if self.antiecho != None:
                        for r in self.antiecho(id):
                            yield r
                    else:
                        yield id

            # reset color to default if we there is no pending user input
            if self.params.input_echo and len(self.embd_inp) == self.input_consumed:
                self.set_color(util.CONSOLE_COLOR_DEFAULT)

            if self.params.interactive and len(self.embd_inp) <= self.input_consumed:
                # if antiprompt is present, stop
                if self.use_antiprompt():
                    if True in [
                        i == self.last_n_tokens[-len(i) :]
                        for i in self.first_antiprompt
                    ]:
                        break

                # if we are using instruction mode, and we have processed the initial prompt
                if self.params.interactive_start:
                    break

            # end of text token
            if len(self.embd) > 0 and self.embd[-1] == llama_cpp.llama_token_eos(
                self.ctx
            ):
                if not self.params.instruct:
                    for i in self.llama_token_eot:
                        yield i
                    break

            # respect n_predict even if antiprompt is present
            if (
                self.params.interactive
                and self.remaining_tokens <= 0
                and self.params.n_predict != -1
            ):
                # If we arent in instruction mode, fix the current generation by appending the antiprompt.
                # Makes it so if chat ends prematurely you dont append the AI's text etc.
                if not self.params.instruct:
                    self.embd_inp += self.first_antiprompt[0]
                self.n_remain = self.params.n_predict
                break

        self.params.interactive_start = False

    def __enter__(self):
        return self

    def __exit__(self, type, value, tb):
        self.exit()

    def exit(self):
        llama_cpp.llama_free(self.ctx)
        self.set_color(util.CONSOLE_COLOR_DEFAULT)

    def token_to_str(self, token_id: int) -> bytes:
        size = 32
        buffer = (ctypes.c_char * size)()
        n = llama_cpp.llama_token_to_piece(
            self.model, llama_cpp.llama_token(token_id), buffer, size
        )
        assert n <= size
        return bytes(buffer[:n])

    # return past text
    def past(self):
        for id in self.last_n_tokens[-self.n_past :]:
            yield self.token_to_str(id).decode("utf8", errors="ignore")

    # write input
    def input(self, prompt: str):
        if (
            self.params.instruct
            and self.last_n_tokens[-len(self.inp_prefix) :] != self.inp_prefix
        ):
            self.embd_inp += self.inp_prefix
        self.embd_inp += self._tokenize(prompt)
        if self.params.instruct:
            self.embd_inp += self.inp_suffix

    # write output
    def output(self):
        self.remaining_tokens = self.params.n_predict
        for id in self.generate():
            cur_char = self.token_to_str(id)

            # Add remainder of missing bytes
            if None in self.multibyte_fix:
                self.multibyte_fix[self.multibyte_fix.index(None)] = cur_char

            # Return completed utf char
            if len(self.multibyte_fix) > 0 and not None in self.multibyte_fix:
                yield (b"".join(self.multibyte_fix)).decode("utf8")
                self.multibyte_fix = []
                continue

            # Contains multi-byte UTF8
            for num, pattern in [(2, 192), (3, 224), (4, 240)]:
                # Bitwise AND check
                if pattern & int.from_bytes(cur_char, "little") == pattern:
                    self.multibyte_fix = [cur_char] + ([None] * (num - 1))

            # Stop incomplete bytes from passing
            if len(self.multibyte_fix) > 0:
                continue

            yield cur_char.decode("utf8")

    # read user input
    def read_input(self):
        out = ""
        while (t := input()).endswith("\\"):
            out += t[:-1] + "\n"
        return out + t + "\n"

    # interactive mode
    def interact(self):
        for i in self.output():
            print(i, end="", flush=True)
        self.params.input_echo = False

        # Using string instead of tokens to check for antiprompt,
        # It is more reliable than tokens for interactive mode.
        generated_str = ""
        while self.params.interactive:
            self.set_color(util.CONSOLE_COLOR_USER_INPUT)
            if self.params.instruct:
                print("\n> ", end="")
                self.input(self.read_input())
            else:
                print(self.params.input_prefix, end="")
                self.input(
                    f"{self.params.input_prefix}{self.read_input()}{self.params.input_suffix}"
                )
                print(self.params.input_suffix, end="")
            self.set_color(util.CONSOLE_COLOR_DEFAULT)

            try:
                for i in self.output():
                    print(i, end="", flush=True)
                    generated_str += i
                    for ap in self.params.antiprompt:
                        if generated_str.endswith(ap):
                            raise KeyboardInterrupt
            except KeyboardInterrupt:
                self.set_color(util.CONSOLE_COLOR_DEFAULT)
                if not self.params.instruct:
                    print(self.params.fix_prefix, end="")
                    self.input(self.params.fix_prefix)


if __name__ == "__main__":
    from datetime import datetime

    USER_NAME = "User"
    AI_NAME = "ChatLLaMa"

    time_now = datetime.now()
    prompt = f"""Text transcript of a never ending dialog, where {USER_NAME} interacts with an AI assistant named {AI_NAME}.
{AI_NAME} is helpful, kind, honest, friendly, good at writing and never fails to answer {USER_NAME}’s requests immediately and with details and precision.
Transcript below contains only the recorded dialog between two, without any annotations like (30 seconds passed...) or (to himself), just what {USER_NAME} and {AI_NAME} say aloud to each other.
The dialog lasts for years, the entirety of it is shared below. It's 10000 pages long.
The transcript only includes text, it does not include markup like HTML and Markdown.

{USER_NAME}: Hello, {AI_NAME}!
{AI_NAME}: Hello {USER_NAME}! How may I help you today?
{USER_NAME}: What time is it?
{AI_NAME}: It is {time_now.strftime("%H:%M")}.
{USER_NAME}: What year is it?
{AI_NAME}: We are in {time_now.strftime("%Y")}.
{USER_NAME}: What is a cat?
{AI_NAME}: A cat is a domestic species of small carnivorous mammal. It is the only domesticated species in the family Felidae.
{USER_NAME}: Name a color.
{AI_NAME}: Blue
{USER_NAME}:   """

    params = gpt_params_parse()
    if params.prompt is None and params.file is None:
        params.prompt = prompt

    with LLaMAInteract(params) as m:
        m.interact()