Advice on how to add Gemma 3 vision support to my code

[jarroddavis68]

Can any one advise me the best way to do this with the way my code works, if possible. The gemma example is structured differently. I'm hoping to not have to redo all my code if possible.

I have to all the models loaded already; this is my code to run inference. I've been working on this all day and it I do not full understand yet the vision code from the gemma3-cli example:

function  TgeInference.Run(const AMessages: TgeMessages; const AMaxContext: Cardinal; const AMaxThreads: Integer): Boolean;
var
  LNumPrompt: Integer;
  LPromptTokens: TArray<llama_token>;
  LCtxParams: llama_context_params;
  LNumPredict: integer;
  LCtx: Pllama_context;
  LSmplrParams: llama_sampler_chain_params;
  LSmplr: Pllama_sampler;
  N: Integer;
  LTokenStr: string;
  LBatch: llama_batch;
  LNewTokenId: llama_token;
  LNumPos: Integer;
  LPrompt: UTF8String;
  LFirstToken: Boolean;
  V: Int32;
  LBuf: array[0..255] of UTF8Char;
  LKey: string;
  LMaxContext: Cardinal;
  LVocab: Pllama_vocab;
begin
  Result := False;

  // check if inference is already runnig
  if FActive then
  begin
    SetError('[%s] Inference already active', ['RunInference']);
    Exit;
  end;

  // check if model not loaded
  if not ModelLoaded() then
  begin
    SetError('[%s] Model not loaded', ['RunInference']);
    Exit;
  end;

  if not AMessages.Empty() then
  begin
    AMessages.AddEnd();
  end;

  FPrompt := AMessages.Prompt();

  if FPrompt.IsEmpty then
  begin
    SetError('Not messages was found', []);
    Exit(False);
  end;

  FActive := True;
  FResponse := '';

  FError := '';
  LFirstToken := True;
  LMaxContext := 0;

  for V := 0 to llama_model_meta_count(FModel)-1 do
  begin
    llama_model_meta_key_by_index(FModel, V, @LBuf[0], length(LBuf));
    LKey := string(LBuf);
    if LKey.Contains('context_length') then
    begin
      llama_model_meta_val_str_by_index(FModel, V, @LBuf[0], length(LBuf));
      LKey := string(LBuf);
      LMaxContext :=  LKey.ToInteger;
      break;
    end;
  end;

  if LMaxContext > 0 then
    LNumPredict := EnsureRange(AMaxContext, 512, LMaxContext)
  else
    LNumPredict := 512;

  LVocab := llama_model_get_vocab(FModel);

  LPrompt := UTF8String(FPrompt);

  LNumPrompt := -llama_tokenize(LVocab, PUTF8Char(LPrompt), Length(LPrompt), nil, 0, true, true);

  SetLength(LPromptTokens, LNumPrompt);

  if llama_tokenize(LVocab, PUTF8Char(LPrompt), Length(LPrompt), @LPromptTokens[0], Length(LPromptTokens), true, true) < 0 then
  begin
    SetError('Failed to tokenize prompt', []);
  end;

  LCtxParams := llama_context_default_params();
  LCtxParams.n_ctx := LNumPrompt + LNumPredict - 1;
  LCtxParams.n_batch := LNumPrompt;
  LCtxParams.no_perf := false;
  if AMaxThreads = -1 then
    LCtxParams.n_threads := geUtils.GetPhysicalProcessorCount()
  else
    LCtxParams.n_threads := EnsureRange(AMaxThreads, 1, geUtils.GetPhysicalProcessorCount());
  LCtxParams.n_threads_batch := LCtxParams.n_threads;
  LCtxParams.flash_attn := False;

  LCtx := llama_new_context_with_model(FModel, LCtxParams);
  if LCtx = nil then
  begin
    SetError('Failed to create inference context', []);
    llama_free_model(FModel);
    exit;
  end;

  LSmplrParams := llama_sampler_chain_default_params();
  LSmplr := llama_sampler_chain_init(LSmplrParams);
  llama_sampler_chain_add(LSmplr, llama_sampler_init_greedy());

  LBatch := llama_batch_get_one(@LPromptTokens[0], Length(LPromptTokens));

  LNumPos := 0;

  FOutputTokens := 0;
  FInputTokens := 0;
  FTokenSpeed := 0;

  OnStart();

  llama_perf_context_reset(LCtx);

  while LNumPos + LBatch.n_tokens < LNumPrompt + LNumPredict do
  begin
    if OnCancel() then Break;

    N := llama_decode(LCtx, LBatch);
    if N <> 0 then
    begin
      SetError('Failed to decode context', []);
      llama_sampler_free(LSmplr);
      llama_free(LCtx);
      llama_free_model(FModel);
      Exit;
    end;

    LNumPos := LNumPos + LBatch.n_tokens;

    LNewTokenId := llama_sampler_sample(LSmplr, LCtx, -1);

    if llama_token_is_eog(LVocab, LNewTokenId) then
    begin
      break;
    end;

    LTokenStr := TokenToPiece(LVocab, LCtx, LNewTokenId, false);

    if LFirstToken then
    begin
      LTokenStr := LTokenStr.Trim();
      LFirstToken := False;
    end;

    case FTokenResponse.AddToken(LTokenStr) of
      tpaWait:
      begin
      end;

      tpaAppend:
      begin
        DoOnNextToken(FTokenResponse.LastWord(False));
      end;

      tpaNewline:
      begin
        DoOnNextToken(#10);
        DoOnNextToken(FTokenResponse.LastWord(True));
      end;
    end;

    LBatch := llama_batch_get_one(@LNewTokenId, 1);
  end;

  if FTokenResponse.Finalize then
  begin
    case FTokenResponse.AddToken('') of
      tpaWait:
      begin
      end;

      tpaAppend:
      begin
        DoOnNextToken(FTokenResponse.LastWord(False));
      end;

      tpaNewline:
      begin
        DoOnNextToken(#10);
        DoOnNextToken(FTokenResponse.LastWord(True));
      end;
    end;
  end;

  OnEnd();

  CalcPerformance(LCtx);

  llama_sampler_free(LSmplr);
  llama_free(LCtx);

  FActive := False;
  FTokenResponse.Clear();

  Result := True;
end;