search_api_ai.module

<?php

/**
 * Implements hook_config_info().
 */
function search_api_ai_config_info() {
  return [
    'search_api_ai.settings' => [
      'label' => t('Search Api Ai Settings'),
      'group' => t('Configuration'),
    ],
  ];
}

/**
 * Implements hook_autoload_info().
 *
 * Autoload classes required for Search API AI.
 */
function search_api_ai_autoload_info() {
  return array(
    'EmbeddingEnginePluginManager' => 'includes/EmbeddingEnginePluginManager.inc',
    'EmbeddingEngineStatic' => 'includes/EmbeddingEngineStatic.inc',
    'EmbeddingEnginePluginBase' => 'includes/EmbeddingEnginePluginBase.inc',
    'EmbeddingEngineInterface' => 'includes/EmbeddingEngineInterface.inc',
    'SearchApiAiBackendInterface' => 'includes/SearchApiAiBackendInterface.inc',
    'TextChunker' => 'includes/TextChunker.inc',
    'EmbeddingEngine' => 'includes/EmbeddingEngine.inc',
    'SearchApiAiBackendPluginBase' => 'includes/SearchApiAiBackendPluginBase.inc',
    'SearchApiAiBackend' => 'includes/SearchApiAiBackend.inc',
  );
}

/**
 * Implements hook_embedding_engine_info().
 */
function search_api_ai_embedding_engine_info() {
  return [
    'openai' => [
      'label' => t('OpenAI Embeddings'),
      'class' => 'OpenAiEmbeddingsDefault',
    ],
    'pinecone' => [
      'label' => t('Pinecone Vector Store'),
      'class' => 'PineconeVectorClient',
    ],
  ];
}

/**
 * Implements hook_search_api_data_type_info().
 */
function search_api_ai_search_api_data_type_info() {
  return array(
    'llm_vector_embedding' => array(
      'name' => t('Embeddings'),
      'fallback' => 'string', // Falls back to string if unsupported by the backend.
      'conversion callback' => 'search_api_ai_convert_embedding',
    ),
  );
}

function search_api_ai_get_embedding_dimension() {
  $search_api_ai_config = config('search_api_ai.settings');

  $embedding_config = $search_api_ai_config->get('embeddings_engine_configuration') ?? [];

  // Ensure we are correctly retrieving the dimension
  watchdog('search_api_ai', '🔍 Retrieved embedding dimension from config: @dimension', [
    '@dimension' => isset($embedding_config['dimension']) ? $embedding_config['dimension'] : 'MISSING',
  ], WATCHDOG_DEBUG);

  return (int) ($embedding_config['dimension'] ?? 1536);
}


function search_api_ai_convert_embedding($value, $original_type, $type, $field_name = 'unknown_field') {
  if ($type !== 'llm_vector_embedding') {
    watchdog('search_api_ai', "⚠️ Unexpected Type: $type", [], WATCHDOG_WARNING);
    return NULL;
  }

  if (!is_string($value)) {
    watchdog('search_api_ai', "❌ Expected a string, received @type for $field_name.", [
      '@type' => gettype($value),
    ], WATCHDOG_ERROR);
    return NULL;
  }

  $cleaned_text = trim(strip_tags($value));
  if (empty($cleaned_text)) {
    watchdog('search_api_ai', "⚠️ Skipping empty text for field: $field_name", [], WATCHDOG_WARNING);
    return NULL;
  }
  $search_api_ai_config = config('search_api_ai.settings');
  // ✅ Get the correct model name
  $model = $search_api_ai_config->get('embeddings_engine');

  if (empty($model)) {
    watchdog('search_api_ai', "❌ No valid embeddings model set.", [], WATCHDOG_ERROR);
    return NULL;
  }

  // ✅ Load OpenAI API settings
  module_load_include('inc', 'openai', 'includes/string_helper');
  $openai_config = config('openai.settings');
  $api_key_name = $openai_config->get('api_key');
  $api_key = key_get_key_value($api_key_name);
  $openai_api = new OpenAIApi($api_key);

  // ✅ Generate embedding
  try {
    $vector = $openai_api->embedding($cleaned_text, $model);

    if (empty($vector) || !is_array($vector)) {
      watchdog('search_api_ai', "❌ Failed to generate embedding for $field_name", [], WATCHDOG_ERROR);
      return NULL;
    }

    // ✅ Ensure correct size
    $expected_dimension = config_get('search_api_ai.settings', 'embeddings_engine_configuration')['dimension'] ?? 1536;
    if (count($vector) !== $expected_dimension) {
      watchdog('search_api_ai', "❌ Invalid vector size for $field_name: Expected $expected_dimension, got " . count($vector), [], WATCHDOG_ERROR);
      return NULL;
    }

    return $vector;

  } catch (Exception $e) {
    watchdog('search_api_ai', "❌ OpenAI API error: " . $e->getMessage(), [], WATCHDOG_ERROR);
    return NULL;
  }
}

/**
 * Implements hook_init().
 *
 * Initializes the plugin manager and stores it globally.
 */
function search_api_ai_init() {
  // Instantiate the plugin manager.
  $plugin_manager = new EmbeddingEnginePluginManager();

  // Instantiate the static helper class, passing the plugin manager.
  $embedding_engine_static = new EmbeddingEngineStatic($plugin_manager);

  // Store it in a global variable if needed.
  $GLOBALS['search_api_ai']['embedding_engine_static'] = $embedding_engine_static;
}


/**
 * Implements hook_menu().
 */
function search_api_ai_menu() {
  $items = array();

  $items['admin/config/search-api-ai/test'] = array(
    'title' => 'Embedding Engine Test Form',
    'description' => 'A test form for embedding engine configuration.',
    'page callback' => 'search_api_ai_embedding_engine_test_form',
    'access arguments' => array('administer site configuration'),
    'type' => MENU_NORMAL_ITEM,
    'parent' => 'admin/config',
  );

  // Route for the embedding engine settings form.
  /*$items['admin/config/search-api-ai/embedding-engine'] = array(
    'title' => 'Embedding Engine Settings',
    'page callback' => 'backdrop_get_form',
    // Again, convert the Drupal form class to a Backdrop form function.
    'page arguments' => array('search_api_ai_embedding_engine_config_form'),
    'access arguments' => array('administer site configuration'),
  );*/

  return $items;
}

/**
 * Implements hook_form_alter().
 */
function search_api_ai_form_alter(&$form, &$form_state, $form_id) {
  if ($form_id == 'search_api_admin_index_edit' || $form_id == 'search_api_admin_add_index') {

    // Add weights to existing form elements
    $form['name']['#weight'] = 10;
    $form['machine_name']['#weight'] = 20;
    $form['item_type']['#weight'] = 30;
    $form['datasource']['#weight'] = 40;
    $form['enabled']['#weight'] = 50;
    $form['description']['#weight'] = 60;
    $form['server']['#weight'] = 70;
    $form['read_only']['#weight'] = 90;
    $form['options']['#weight'] = 100;
    $form['submit']['#weight'] = 110;

    // Load existing configuration
    $config = config_get('search_api_ai.settings');

    // Get the saved settings or default values
    $settings = [];
    if ($index) {
      $settings = !empty($index->options['embedding_settings'])
        ? $index->options['embedding_settings']
        : [];
    }

    // Set default values from saved settings
    $selected_engine = isset($settings['embeddings_engine'])
      ? $settings['embeddings_engine']
      : '';


    // Load available embedding engines
    $all_engines = module_invoke_all('embedding_engine_info');

    // Filter valid embedding engines
    $embedding_engines = array_filter($all_engines, function ($engine) {
      return isset($engine['class'])
        && class_exists($engine['class'])
        && is_subclass_of($engine['class'], 'OpenAiEmbeddingsDefault');
    });

    // Build options array using model names
    $options = [];
    foreach ($embedding_engines as $id => $engine) {
      $engine_class = $engine['class'];
      if (class_exists($engine_class)) {
        try {
          $engine_instance = new $engine_class([], NULL);
          $options[$id] = method_exists($engine_instance, 'getModelName')
            ? $engine_instance->getModelName()
            : $id;
        }
        catch (Exception $e) {
          watchdog('search_api_ai', 'Error instantiating embedding engine @engine: @message', [
            '@engine' => $id,
            '@message' => $e->getMessage(),
          ], WATCHDOG_ERROR);
        }
      }
    }

    // Get dimension value
    $engine_config = $config['embeddings_engine_configuration'] ?? ['dimension' => 768];
    $dimension_value = $engine_config['dimension'] ?? 768;

    // Add the embedding engine selection with states
    $form['embeddings_engine'] = [
      '#type' => 'select',
      '#title' => t('Embedding Engine'),
      '#options' => $options,
      '#default_value' => $selected_engine,
      '#description' => t('Select the embedding engine to use. This needs to match the setting in the Vector database.'),
      '#weight' => 75,
      '#ajax' => [
        'callback' => 'search_api_ai_update_embedding_dimension',
        'wrapper' => 'embedding-dimension-wrapper',
        'event' => 'change',
      ],
      '#states' => [
        'visible' => [
          'select[name="server"]' => [
            ['value' => 'pinecone'],
            ['value' => 'milvus'],
          ],
        ],
      ],
    ];

    // Add the configuration fieldset with states
    $form['embeddings_engine_configuration'] = [
      '#type' => 'fieldset',
      '#title' => t('Embedding Engine Configuration'),
      '#prefix' => '<div id="embedding-dimension-wrapper">',
      '#suffix' => '</div>',
      '#weight' => 80,
      '#states' => [
        'visible' => [
          'select[name="server"]' => [
            ['value' => 'pinecone'],
            ['value' => 'milvus'],
          ],
        ],
      ],
    ];

    $form['embeddings_engine_configuration']['dimension'] = [
      '#type' => 'textfield',
      '#title' => t('Embedding Dimension'),
      '#default_value' => $dimension_value,
      '#description' => t('The number of dimensions for embeddings.'),
    ];

    // Add AJAX to the server selection
    $form['server']['#ajax'] = [
      'callback' => 'search_api_ai_server_selection_callback',
      'wrapper' => 'edit-wrapper', // Ensure this matches your form wrapper ID
      'method' => 'replace',
    ];
  }
}

/**
 * AJAX callback for server selection.
 */
function search_api_ai_server_selection_callback($form, &$form_state) {
  return $form;
}

/**
 * Renders the embedding engine test form.
 *
 * @return array
 *   The renderable form array.
 */
function search_api_ai_embedding_engine_test_form() {
  // Load helper functions from includes.
  module_load_include('inc', 'search_api_ai', 'includes/SearchApiAiBackend');

  // Mock configuration for testing.
  $configuration = array(
    'embeddings_engine' => NULL,
    'embeddings_engine_configuration' => array(
      'dimension' => 768,
    ),
  );

  // Initialize form and form state.
  $form = array();
  $form_state = array();  // Initialize form state as an array.

  // Pass $form_state by reference.
  $form = search_api_ai_engine_configuration_form($form, $form_state, $configuration);

  return $form;
}

/**
 * Build the configuration form for selecting and configuring the embedding engine.
 */
function search_api_ai_embedding_engine_config_form($form, &$form_state) {
  // Load existing configuration.
  $config = config_get('search_api_ai.settings');

  $selected_engine = $form_state['values']['embeddings_engine'] ??
    ($config['embeddings_engine'] ?? '');

  // Default embedding dimension.
  $engine_config = $config['embeddings_engine_configuration'] ?? ['dimension' => 768];

  // Load available embedding engines.
  $all_engines = module_invoke_all('embedding_engine_info');

  // Filter only valid embedding engines dynamically.
  $embedding_engines = array_filter($all_engines, function ($engine) {
    return isset($engine['class']) && class_exists($engine['class']) && is_subclass_of($engine['class'], 'OpenAiEmbeddingsDefault');
  });

  // Populate the form options using **model name from the class**.
  $options = [];
  foreach ($embedding_engines as $id => $engine) {
    $engine_class = $engine['class'];
    if (class_exists($engine_class)) {
      try {
        $engine_instance = new $engine_class([], NULL);
        $options[$id] = method_exists($engine_instance, 'getModelName') ?
          $engine_instance->getModelName() : $id;
      } catch (Exception $e) {
        watchdog('search_api_ai', 'Error instantiating embedding engine @engine: @message', [
          '@engine' => $id,
          '@message' => $e->getMessage(),
        ], WATCHDOG_ERROR);
      }
    }
  }

  // Initialize dimension with a fallback value.
  $dimension_value = $engine_config['dimension'] ?? 768;

  // Set form elements.
  $form['embeddings_engine'] = [
    '#type' => 'select',
    '#title' => t('Embedding Engine'),
    '#options' => $options,
    '#default_value' => $selected_engine,
    '#description' => t('Select the embedding engine to use.'),
    '#ajax' => [
      'callback' => 'search_api_ai_update_embedding_dimension',
      'wrapper' => 'embedding-dimension-wrapper', // This should match the fieldset wrapper
      'event' => 'change',
    ],
  ];

  $form['embeddings_engine_configuration'] = [
    '#type' => 'fieldset',
    '#title' => t('Embedding Engine Configuration'),
    '#prefix' => '<div id="embedding-dimension-wrapper">',
    '#suffix' => '</div>',
  ];

  $form['embeddings_engine_configuration']['dimension'] = [
    '#type' => 'textfield',
    '#title' => t('Embedding Dimension'),
    '#default_value' => $dimension_value,
    '#description' => t('The number of dimensions for embeddings.'),
  ];

  $form['actions'] = ['#type' => 'actions'];
  $form['actions']['submit'] = [
    '#type' => 'submit',
    '#value' => t('Save Configuration'),
  ];

  return $form;
}

/**
 * AJAX Callback to update embedding dimension dynamically.
 */
function search_api_ai_update_embedding_dimension($form, &$form_state) {
  $selected_engine = $form_state['values']['embeddings_engine'] ?? NULL;
  $dimension_value = 768; // Default

  // Load available embedding engines.
  $all_engines = module_invoke_all('embedding_engine_info');

  if ($selected_engine && isset($all_engines[$selected_engine])) {
    $engine_class = $all_engines[$selected_engine]['class'];

    if (class_exists($engine_class)) {
      try {
        // Retrieve OpenAI API key from configuration.
        $api_key_name = config_get('openai.settings', 'api_key');
        $api_key = key_get_key_value($api_key_name);

        if (!$api_key) {
          throw new Exception('OpenAI API key is missing.');
        }

        // Create OpenAI client instance.
        $client = new OpenAIApi($api_key);

        // Create embedding engine instance with correct arguments.
        $engine_instance = new $engine_class(config_get('search_api_ai.settings'), $client);

        // Get the dimension value dynamically.
        if (method_exists($engine_instance, 'getDimension')) {
          $dimension_value = $engine_instance->getDimension();
        }
      } catch (Exception $e) {
        watchdog('search_api_ai', 'Failed to update dimension for @engine: @message', [
          '@engine' => $selected_engine,
          '@message' => $e->getMessage(),
        ], WATCHDOG_ERROR);
      }
    }
  }

  // Update the dimension field dynamically.
  $form['embeddings_engine_configuration']['dimension']['#value'] = $dimension_value;

  return $form['embeddings_engine_configuration'];
}

/**
 * Submit handler for the embedding engine configuration form.
 */
function search_api_ai_embedding_engine_config_form_submit($form, &$form_state) {
  $selected_engine = $form_state['values']['embeddings_engine'] ?? '';
  $engine_config = $form_state['values']['embeddings_engine_configuration'] ?? ['dimension' => 768];

  // Load available embedding engines.
  $all_engines = module_invoke_all('embedding_engine_info');

  if ($selected_engine && isset($all_engines[$selected_engine])) {
    $engine_class = $all_engines[$selected_engine]['class'];

    if (class_exists($engine_class)) {
      try {
        // Retrieve OpenAI API key.
        $api_key_name = config_get('openai.settings', 'api_key');
        $api_key = key_get_key_value($api_key_name);
        if (!$api_key) {
          throw new Exception('OpenAI API key is missing.');
        }

        // Create OpenAI client instance.
        $client = new OpenAIApi($api_key);
        $engine_instance = new $engine_class(config_get('search_api_ai.settings'), $client);

        // Retrieve **clean model name** from the class.
        $model_name = method_exists($engine_instance, 'getModelName')
          ? $engine_instance->getModelName()
          : $selected_engine;

        // Retrieve dimension from the selected engine.
        if (method_exists($engine_instance, 'getDimension')) {
          $engine_config['dimension'] = $engine_instance->getDimension();
        }
      } catch (Exception $e) {
        backdrop_set_message(t('Failed to load embedding engine: @message', [
          '@message' => $e->getMessage(),
        ]), 'error');
        watchdog('search_api_ai', 'Error initializing embedding engine @engine: @message', [
          '@engine' => $selected_engine,
          '@message' => $e->getMessage(),
        ], WATCHDOG_ERROR);
      }
    }
  }

  // Save the configuration using **model name instead of engine ID**.
  config_set('search_api_ai.settings', 'embeddings_engine', $model_name);
  config_set('search_api_ai.settings', 'embeddings_engine_configuration', $engine_config);

  backdrop_set_message(t('Embedding engine configuration has been saved.'));
}

/**
 * Implements hook_search_api_index_items_alter().
 *
 * This hook alters items during indexing to set the embedding engine.
 *
 * @param array $items
 *   The items being indexed.
 * @param object $index
 *   The index object.
 */
function search_api_ai_search_api_index_items_alter(array &$items, SearchApiIndex $index) {
  global $search_api_ai_vectors;

  if (!isset($search_api_ai_vectors)) {
    $search_api_ai_vectors = [];
  }

  watchdog('search_api_ai', '🔄 Altering index items for @index', [
    '@index' => $index->machine_name
  ], WATCHDOG_NOTICE);

  // ✅ Load Search API index configuration
  $query = db_select('search_api_index', 's')
    ->fields('s', ['options'])
    ->condition('machine_name', $index->machine_name)
    ->execute()
    ->fetchField();

  if (!$query) {
    watchdog('search_api_ai', "❌ Could not load index configuration.", [], WATCHDOG_ERROR);
    return;
  }

  // ✅ Extract field settings from index options
  $index_options = unserialize($query);
  if (empty($index_options['fields']) || !is_array($index_options['fields'])) {
    watchdog('search_api_ai', "❌ No valid fields found in index configuration.", [], WATCHDOG_ERROR);
    return;
  }

  // ✅ Identify fields that need embedding
  $embedding_fields = [];
  foreach ($index_options['fields'] as $field_name => $field_info) {
    if (!empty($field_info['real_type']) && $field_info['real_type'] === 'llm_vector_embedding') {
      $embedding_fields[] = $field_name;
    }
  }

  // ✅ Load configured embedding dimension from settings
  $search_api_ai_config = config('search_api_ai.settings');
  $expected_dimension = $search_api_ai_config->get('embeddings_engine_configuration')['dimension'] ?? 1536;

  watchdog('search_api_ai', "🛠 Using embedding dimension: @dimension", [
    '@dimension' => $expected_dimension
  ], WATCHDOG_DEBUG);

  // Process each entity for embedding
  foreach ($items as $id => &$entity) {
    if (!is_object($entity)) {
      continue;
    }

    // ✅ Correctly Extract nid['value']
    $nid = isset($entity->nid) ? $entity->nid : NULL;

    if (empty($nid)) {
      watchdog('search_api_ai', '⚠️ Skipping entity @id due to missing nid. Raw nid: <pre>@nid</pre>', [
        '@id' => $id,
        '@nid' => print_r($entity->nid, TRUE),
      ], WATCHDOG_WARNING);
      continue;
    }

    // ✅ Process each field independently
    $field_embeddings = [];

    foreach ($embedding_fields as $field_name) {
      $real_field_name = ($field_name === 'body:value') ? 'body' : $field_name;
      $field_texts = [];

      if (!empty($entity->{$real_field_name})) {
        if (is_array($entity->{$real_field_name})) {
          foreach ($entity->{$real_field_name} as $lang => $values) {
            if (is_array($values)) {
              foreach ($values as $delta => $value_data) {
                if (!empty($value_data['value']) && is_string($value_data['value'])) {
                  $cleaned_text = trim(strip_tags($value_data['value']));
                  if (!empty($cleaned_text)) {
                    $field_texts[] = $cleaned_text;
                  }
                }
              }
            }
          }
        } else {
          $cleaned_text = trim(strip_tags($entity->{$real_field_name}));
          if (!empty($cleaned_text)) {
            $field_texts[] = $cleaned_text;
          }
        }
      }

      foreach ($field_texts as $text) {
        if (empty($text)) {
          watchdog('search_api_ai', "⚠️ Skipping empty text for $real_field_name.", [], WATCHDOG_WARNING);
          continue;
        }

        $converted_value = search_api_ai_convert_embedding($text, 'string', 'llm_vector_embedding', $real_field_name);

        if (!empty($converted_value)) {
          // ✅ Ensure the embedding has the correct size
          if (!is_array($converted_value) || count($converted_value) !== $expected_dimension) {
            watchdog('search_api_ai', "❌ Invalid vector size for $real_field_name in item $nid: Expected $expected_dimension, got " . count($converted_value), [], WATCHDOG_ERROR);
            continue;
          }

          $field_embeddings[$real_field_name] = [
            'content' => $text,
            'vectors' => $converted_value,
          ];
        } else {
          watchdog('search_api_ai', "❌ Failed to generate embedding for $real_field_name", [], WATCHDOG_ERROR);
        }
      }
    }

    // ✅ Store embeddings in the global vector storage
    if (!empty($field_embeddings)) {
      $search_api_ai_vectors[$nid] = [
        'nid' => $nid,
        'vectors' => $field_embeddings,
      ];

      watchdog('search_api_ai', '🛠 Final vectors stored for nid @nid: <pre>@vectors</pre>', [
        '@nid' => $nid,
        '@vectors' => print_r($search_api_ai_vectors[$nid], TRUE),
      ], WATCHDOG_DEBUG);
    }
  }

  watchdog('search_api_ai', '✅ Finished altering index items for @index', [
    '@index' => $index->machine_name
  ], WATCHDOG_NOTICE);
}