diff --git a/.env.example b/.env.example index 916c979a7..2e694fc8f 100644 --- a/.env.example +++ b/.env.example @@ -22,6 +22,7 @@ USE_CHARACTER_STORAGE=false # Logging DEFAULT_LOG_LEVEL=warn LOG_JSON_FORMAT=false # Print everything in logger as json; false by default +INSTRUMENTATION_ENABLED=false # Requires to define POSTGRES_URL ############################### #### Client Configurations #### diff --git a/agent/src/index.ts b/agent/src/index.ts index 8ee47854b..056fa9d5e 100644 --- a/agent/src/index.ts +++ b/agent/src/index.ts @@ -20,6 +20,7 @@ import { type IDatabaseAdapter, type IDatabaseCacheAdapter, ModelProviderName, + type State, defaultCharacter, elizaLogger, parseBooleanFromText, @@ -52,7 +53,10 @@ import { import Database from "better-sqlite3"; import yargs from "yargs"; import { z } from "zod"; -import { getRuntimeInstrumentation } from "./runtime-instrumentation"; +import { + type RuntimeInstrumentation, + getRuntimeInstrumentation, +} from "./runtime-instrumentation"; const __filename = fileURLToPath(import.meta.url); // get the resolved path to the file const __dirname = path.dirname(__filename); // get the name of the directory @@ -580,14 +584,17 @@ export async function createAgent( fetch: logFetch, }); - // Set up automatic instrumentation for all agents - const runtimeInstrumentation = getRuntimeInstrumentation(); + if (process.env.INSTRUMENTATION_ENABLED === "true") { + // Set up automatic instrumentation for all agents + const runtimeInstrumentation = getRuntimeInstrumentation(); - // This will attach to evaluate and other methods to automatically instrument - runtimeInstrumentation.attachToRuntime(runtime); - elizaLogger.info( - `🔄 Instrumentation attached to runtime for agent ${runtime.agentId}`, - ); + // This will attach to evaluate and other methods to automatically instrument + // biome-ignore lint/suspicious/noExplicitAny: + runtimeInstrumentation.attachToRuntime(runtime as any); + elizaLogger.info( + `🔄 Instrumentation attached to runtime for agent ${runtime.agentId}`, + ); + } return runtime; } @@ -664,11 +671,9 @@ async function startAgent( }; // Call evaluate to trigger instrumentation - runtime - .evaluate(message, state as Record) - .catch((err) => { - elizaLogger.error(`Error evaluating agent start: ${err.message}`); - }); + runtime.evaluate(message, state as State).catch((err) => { + elizaLogger.error(`Error evaluating agent start: ${err.message}`); + }); } catch (error) { elizaLogger.error( `Error during agent start instrumentation: ${error.message}`, @@ -750,9 +755,12 @@ const startAgents = async () => { directClient.loadCharacterTryPath = loadCharacterTryPath; directClient.jsonToCharacter = jsonToCharacter; - // Make sure Runtime Instrumentation is exposed to DirectClient - const runtimeInstrumentation = getRuntimeInstrumentation(); - directClient.instrumentationAttached = false; + let runtimeInstrumentation: RuntimeInstrumentation; + if (process.env.INSTRUMENTATION_ENABLED === "true") { + // Make sure Runtime Instrumentation is exposed to DirectClient + runtimeInstrumentation = getRuntimeInstrumentation(); + directClient.instrumentationAttached = false; + } // Add a method to DirectClient to ensure instrumentation is set up properly const originalRegisterAgent = directClient.registerAgent.bind(directClient); @@ -761,14 +769,18 @@ const startAgents = async () => { originalRegisterAgent(runtime); // Ensure the runtime is instrumented - if (!directClient.instrumentationAttached) { + if ( + process.env.INSTRUMENTATION_ENABLED === "true" && + !directClient.instrumentationAttached + ) { try { // Attach the instrumentation wrapper to all POST request handlers // Apply to each route handler that processes messages elizaLogger.info( `🔌 Attaching instrumentation to DirectClient routes for agent ${runtime.agentId}`, ); - runtimeInstrumentation.attachToRuntime(runtime); + // biome-ignore lint/suspicious/noExplicitAny: + runtimeInstrumentation.attachToRuntime(runtime as any); directClient.instrumentationAttached = true; } catch (error) { elizaLogger.error( @@ -811,7 +823,10 @@ if ( } // Initialize the runtime instrumentation at the module level -const runtimeInstrumentation = getRuntimeInstrumentation(); +let runtimeInstrumentation: RuntimeInstrumentation; +if (process.env.INSTRUMENTATION_ENABLED === "true") { + runtimeInstrumentation = getRuntimeInstrumentation(); +} // Export the instrumentation for direct access if needed export { runtimeInstrumentation }; diff --git a/agent/src/instrumentation.ts b/agent/src/instrumentation.ts index b819e394c..55ea8b3ec 100644 --- a/agent/src/instrumentation.ts +++ b/agent/src/instrumentation.ts @@ -292,4 +292,10 @@ export class Instrumentation { } } -export const instrument = Instrumentation.getInstance(); +let instrument: Instrumentation; + +if (process.env.INSTRUMENTATION_ENABLED === "true") { + instrument = Instrumentation.getInstance(); +} + +export { instrument }; diff --git a/clients/client-direct/src/index.ts b/clients/client-direct/src/index.ts index b13814167..78f0c15dd 100644 --- a/clients/client-direct/src/index.ts +++ b/clients/client-direct/src/index.ts @@ -10,6 +10,7 @@ import { type Media, type Memory, ModelClass, + type State, composeContext, elizaLogger, generateCaption, @@ -21,7 +22,11 @@ import { settings, stringToUuid, } from "@elizaos/core"; -import type { RuntimeLike } from "@realityspiral/agent/runtime-instrumentation"; +import type { Instrumentation } from "@realityspiral/agent/instrumentation"; +import type { + RuntimeInstrumentation, + RuntimeLike, +} from "@realityspiral/agent/runtime-instrumentation"; import bodyParser from "body-parser"; import cors from "cors"; import express, { type Request as ExpressRequest } from "express"; @@ -280,17 +285,20 @@ export class DirectClient { return; } - // Import the runtime instrumentation and instrumentation singleton - const instrumentationModule = await import( - "@realityspiral/agent/instrumentation" - ); - const runtimeInstrumentationModule = await import( - "@realityspiral/agent/runtime-instrumentation" - ); - const runtimeInstrumentation = - runtimeInstrumentationModule.getRuntimeInstrumentation(); - const instrumentation = - instrumentationModule.Instrumentation.getInstance(); + let runtimeInstrumentation: RuntimeInstrumentation; + let instrumentation: Instrumentation; + if (process.env.INSTRUMENTATION_ENABLED === "true") { + // Import the runtime instrumentation and instrumentation singleton + const instrumentationModule = await import( + "@realityspiral/agent/instrumentation" + ); + const runtimeInstrumentationModule = await import( + "@realityspiral/agent/runtime-instrumentation" + ); + runtimeInstrumentation = + runtimeInstrumentationModule.getRuntimeInstrumentation(); + instrumentation = instrumentationModule.Instrumentation.getInstance(); + } const text = req.body.text; // if empty text, directly return @@ -299,10 +307,12 @@ export class DirectClient { return; } - // Ensure the runtime has instrumentation attached - runtimeInstrumentation.attachToRuntime( - runtime as unknown as RuntimeLike, - ); + if (process.env.INSTRUMENTATION_ENABLED === "true") { + // Ensure the runtime has instrumentation attached + runtimeInstrumentation.attachToRuntime( + runtime as unknown as RuntimeLike, + ); + } const startTime = Date.now(); const messageId = stringToUuid(Date.now().toString()); @@ -312,24 +322,26 @@ export class DirectClient { (runtime as unknown as RuntimeLike).sessionId || `session-${Date.now()}`; - // Log message reception event with detailed context - instrumentation.logEvent({ - stage: "Chat", - subStage: "Request", - event: "chat_message_received", - data: { - messageId, - sessionId, - agentId: runtime.agentId, - roomId, - userId, - messageText: text, - characterName: runtime.character?.name || "Unknown", - timestamp: startTime, - client: "direct", - hasAttachments: !!req.file, - }, - }); + if (process.env.INSTRUMENTATION_ENABLED === "true") { + // Log message reception event with detailed context + instrumentation.logEvent({ + stage: "Chat", + subStage: "Request", + event: "chat_message_received", + data: { + messageId, + sessionId, + agentId: runtime.agentId, + roomId, + userId, + messageText: text, + characterName: runtime.character?.name || "Unknown", + timestamp: startTime, + client: "direct", + hasAttachments: !!req.file, + }, + }); + } await runtime.ensureConnection( userId, @@ -399,73 +411,79 @@ export class DirectClient { createdAt: Date.now(), }; - // Log memory storage event - instrumentation.logEvent({ - stage: "Chat", - subStage: "Memory", - event: "memory_storage_started", - data: { - messageId, - sessionId, - agentId: runtime.agentId, - roomId, - userId, - timestamp: Date.now(), - }, - }); + if (process.env.INSTRUMENTATION_ENABLED === "true") { + // Log memory storage event + instrumentation.logEvent({ + stage: "Chat", + subStage: "Memory", + event: "memory_storage_started", + data: { + messageId, + sessionId, + agentId: runtime.agentId, + roomId, + userId, + timestamp: Date.now(), + }, + }); + } await runtime.messageManager.addEmbeddingToMemory(memory); await runtime.messageManager.createMemory(memory); - // Log memory storage complete event - instrumentation.logEvent({ - stage: "Chat", - subStage: "Memory", - event: "memory_storage_completed", - data: { - messageId, - sessionId, - agentId: runtime.agentId, - roomId, - userId, - timestamp: Date.now(), - }, - }); + if (process.env.INSTRUMENTATION_ENABLED === "true") { + // Log memory storage complete event + instrumentation.logEvent({ + stage: "Chat", + subStage: "Memory", + event: "memory_storage_completed", + data: { + messageId, + sessionId, + agentId: runtime.agentId, + roomId, + userId, + timestamp: Date.now(), + }, + }); - // Log state composition started - instrumentation.logEvent({ - stage: "Chat", - subStage: "State", - event: "state_composition_started", - data: { - messageId, - sessionId, - agentId: runtime.agentId, - roomId, - userId, - timestamp: Date.now(), - }, - }); + // Log state composition started + instrumentation.logEvent({ + stage: "Chat", + subStage: "State", + event: "state_composition_started", + data: { + messageId, + sessionId, + agentId: runtime.agentId, + roomId, + userId, + timestamp: Date.now(), + }, + }); + } let state = await runtime.composeState(userMessage, { agentName: runtime.character.name, }); - // Log response generation started - instrumentation.logEvent({ - stage: "Chat", - subStage: "Generation", - event: "response_generation_started", - data: { - messageId, - sessionId, - agentId: runtime.agentId, - roomId, - userId, - modelClass: ModelClass.LARGE, - timestamp: Date.now(), - }, - }); + if (process.env.INSTRUMENTATION_ENABLED === "true") { + // Log response generation started + instrumentation.logEvent({ + stage: "Chat", + subStage: "Generation", + event: "response_generation_started", + data: { + messageId, + sessionId, + agentId: runtime.agentId, + roomId, + userId, + modelClass: ModelClass.LARGE, + timestamp: Date.now(), + }, + }); + } const generationStartTime = Date.now(); const response = await generateMessageResponse({ @@ -478,39 +496,43 @@ export class DirectClient { }); const generationEndTime = Date.now(); - // Log response generation completed - instrumentation.logEvent({ - stage: "Chat", - subStage: "Generation", - event: "response_generation_completed", - data: { - messageId, - sessionId, - agentId: runtime.agentId, - roomId, - userId, - generationTime: generationEndTime - generationStartTime, - responseLength: response ? response.text.length : 0, - timestamp: generationEndTime, - }, - }); - - if (!response) { - // Log error + if (process.env.INSTRUMENTATION_ENABLED === "true") { + // Log response generation completed instrumentation.logEvent({ stage: "Chat", - subStage: "Error", - event: "response_generation_failed", + subStage: "Generation", + event: "response_generation_completed", data: { messageId, sessionId, agentId: runtime.agentId, roomId, userId, - error: "No response from generateMessageResponse", - timestamp: Date.now(), + generationTime: generationEndTime - generationStartTime, + responseLength: response ? response.text.length : 0, + timestamp: generationEndTime, }, }); + } + + if (!response) { + if (process.env.INSTRUMENTATION_ENABLED === "true") { + // Log error + instrumentation.logEvent({ + stage: "Chat", + subStage: "Error", + event: "response_generation_failed", + data: { + messageId, + sessionId, + agentId: runtime.agentId, + roomId, + userId, + error: "No response from generateMessageResponse", + timestamp: Date.now(), + }, + }); + } res.status(500).send("No response from generateMessageResponse"); return; } @@ -531,21 +553,23 @@ export class DirectClient { let message = null as Content | null; - // Log action processing started - instrumentation.logEvent({ - stage: "Chat", - subStage: "Actions", - event: "action_processing_started", - data: { - messageId, - sessionId, - agentId: runtime.agentId, - roomId, - userId, - actionCount: runtime.actions.length, - timestamp: Date.now(), - }, - }); + if (process.env.INSTRUMENTATION_ENABLED === "true") { + // Log action processing started + instrumentation.logEvent({ + stage: "Chat", + subStage: "Actions", + event: "action_processing_started", + data: { + messageId, + sessionId, + agentId: runtime.agentId, + roomId, + userId, + actionCount: runtime.actions.length, + timestamp: Date.now(), + }, + }); + } const actionsStartTime = Date.now(); await runtime.processActions( @@ -559,22 +583,24 @@ export class DirectClient { ); const actionsEndTime = Date.now(); - // Log action processing completed - instrumentation.logEvent({ - stage: "Chat", - subStage: "Actions", - event: "action_processing_completed", - data: { - messageId, - sessionId, - agentId: runtime.agentId, - roomId, - userId, - processingTime: actionsEndTime - actionsStartTime, - hasAdditionalMessages: message !== null, - timestamp: actionsEndTime, - }, - }); + if (process.env.INSTRUMENTATION_ENABLED === "true") { + // Log action processing completed + instrumentation.logEvent({ + stage: "Chat", + subStage: "Actions", + event: "action_processing_completed", + data: { + messageId, + sessionId, + agentId: runtime.agentId, + roomId, + userId, + processingTime: actionsEndTime - actionsStartTime, + hasAdditionalMessages: message !== null, + timestamp: actionsEndTime, + }, + }); + } await runtime.evaluate(memory, state); @@ -582,24 +608,26 @@ export class DirectClient { const action = runtime.actions.find((a) => a.name === response.action); const shouldSuppressInitialMessage = action?.suppressInitialMessage; - // Log response delivery - instrumentation.logEvent({ - stage: "Chat", - subStage: "Response", - event: "response_delivered", - data: { - messageId, - sessionId, - agentId: runtime.agentId, - roomId, - userId, - agentResponse: response.text, - totalProcessingTime: Date.now() - startTime, - suppressedInitialMessage: !!shouldSuppressInitialMessage, - hasAdditionalMessages: message !== null, - timestamp: Date.now(), - }, - }); + if (process.env.INSTRUMENTATION_ENABLED === "true") { + // Log response delivery + instrumentation.logEvent({ + stage: "Chat", + subStage: "Response", + event: "response_delivered", + data: { + messageId, + sessionId, + agentId: runtime.agentId, + roomId, + userId, + agentResponse: response.text, + totalProcessingTime: Date.now() - startTime, + suppressedInitialMessage: !!shouldSuppressInitialMessage, + hasAdditionalMessages: message !== null, + timestamp: Date.now(), + }, + }); + } if (!shouldSuppressInitialMessage) { if (message) { @@ -1308,13 +1336,11 @@ export class DirectClient { // This will trigger the instrumentation if it's properly set up if (runtime.evaluate && typeof runtime.evaluate === "function") { const state = { agentId: runtime.agentId }; - runtime - .evaluate(message, state as Record) - .catch((err) => { - elizaLogger.error( - `Error evaluating agent registration: ${err.message}`, - ); - }); + runtime.evaluate(message, state as State).catch((err) => { + elizaLogger.error( + `Error evaluating agent registration: ${err.message}`, + ); + }); } } catch (error) { elizaLogger.error( diff --git a/scripts/deploy.sh b/scripts/deploy.sh index e4ecb2dee..48ac5cc8a 100644 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -130,28 +130,6 @@ ensure_container_database() { echo "${container_db}" } -# Function to run SQL script for a container -run_sql_for_container() { - local container_name=$1 - - # Ensure container database exists and get its name - local container_db=$(ensure_container_database "${container_name}") - - log_message "Running tracing-schema.sql for ${container_name} in database ${container_db}..." - - # Create a temporary file that customizes the SQL for this container - local temp_sql_file=$(mktemp) - cat tracing-schema.sql | sed "s/{{container_name}}/${container_name}/g" > ${temp_sql_file} - - # Run the SQL script against the container's database - docker exec -i ${PG_CONTAINER_NAME} psql -U ${PG_USER} -d ${container_db} < ${temp_sql_file} - - # Clean up - rm ${temp_sql_file} - - log_message "SQL script execution completed for ${container_name} in database ${container_db}." -} - # Function to get the specific version tag from an image get_version_tag() { local image_name=$1 @@ -248,9 +226,6 @@ deploy_containers() { --network=${DOCKER_NETWORK} \ --restart unless-stopped \ ghcr.io/sifchain/realityspiral:${image_tag} - - # Run SQL script for this container - run_sql_for_container "${container}" done } diff --git a/scripts/tracing-schema.sql b/scripts/tracing-schema.sql deleted file mode 100644 index 683e8e6b8..000000000 --- a/scripts/tracing-schema.sql +++ /dev/null @@ -1,84 +0,0 @@ --- Drop tables if they already exist (drop events first since it depends on traces) -DROP TABLE IF EXISTS events; -DROP TABLE IF EXISTS traces; - --- Create the traces table -CREATE TABLE IF NOT EXISTS traces ( - trace_id VARCHAR(256) NOT NULL, - span_id VARCHAR(256) NOT NULL, - parent_span_id VARCHAR(256), - trace_state VARCHAR(256), - span_name VARCHAR(256) NOT NULL, - span_kind VARCHAR(64) NOT NULL, -- e.g., "INTERNAL", "SERVER", "CLIENT" - start_time TIMESTAMPTZ NOT NULL, - end_time TIMESTAMPTZ NOT NULL, - duration_ms INTEGER, - status_code VARCHAR(64), -- e.g., "OK", "ERROR" - status_message TEXT, - attributes JSONB, - events JSONB, - links JSONB, - resource JSONB, - agent_id VARCHAR(256), - session_id VARCHAR(256), - environment VARCHAR(64), - room_id VARCHAR(256), - raw_context TEXT NOT NULL DEFAULT '', - raw_response TEXT NOT NULL DEFAULT '', - PRIMARY KEY (trace_id, span_id) -); - --- Create indexes for traces table -CREATE INDEX idx_traces_trace_id ON traces (trace_id); -CREATE INDEX idx_traces_span_name ON traces (span_name); -CREATE INDEX idx_traces_start_time ON traces (start_time); -CREATE INDEX idx_traces_room ON traces (room_id); - --- Create the events table with a composite foreign key referencing traces -CREATE TABLE IF NOT EXISTS events ( - event_id UUID DEFAULT gen_random_uuid(), - trace_id VARCHAR(256) NOT NULL, - span_id VARCHAR(256) NOT NULL, - agent_id VARCHAR(256) NOT NULL, - event_type VARCHAR(64) NOT NULL, - event_time TIMESTAMPTZ NOT NULL DEFAULT NOW(), - event_data JSONB NOT NULL, - room_id VARCHAR(256) NOT NULL, - PRIMARY KEY (event_id), - FOREIGN KEY (trace_id, span_id) REFERENCES traces(trace_id, span_id) -); - --- Create indexes for events table -CREATE INDEX idx_events_agent ON events (agent_id); -CREATE INDEX idx_events_type ON events (event_type); -CREATE INDEX idx_events_time ON events (event_time); -CREATE INDEX idx_events_room ON events (room_id); - --- Remove strict constraints temporarily -ALTER TABLE traces -DROP CONSTRAINT IF EXISTS raw_context_not_empty, -DROP CONSTRAINT IF EXISTS raw_response_not_empty; - --- Keep NOT NULL but allow empty strings -ALTER TABLE traces -ALTER COLUMN raw_context DROP DEFAULT, -ALTER COLUMN raw_response DROP DEFAULT; - --- Add smarter constraints that allow placeholders -ALTER TABLE traces -DROP CONSTRAINT valid_raw_context, -DROP CONSTRAINT valid_raw_response; - -ALTER TABLE traces -ADD CONSTRAINT valid_raw_context -CHECK ( - (span_name = 'llm_context_pre' AND raw_context <> '') - OR (span_name <> 'llm_context_pre') -); - -ALTER TABLE traces -ADD CONSTRAINT valid_raw_response -CHECK ( - (span_name = 'llm_response_post' AND raw_response <> '') - OR (span_name <> 'llm_response_post') -); \ No newline at end of file diff --git a/ui/src/components/app-sidebar.tsx b/ui/src/components/app-sidebar.tsx index d5de70f64..8ccdba9bd 100644 --- a/ui/src/components/app-sidebar.tsx +++ b/ui/src/components/app-sidebar.tsx @@ -104,20 +104,24 @@ export function AppSidebar() { - - - - Settings - - - - - - - Templates - - - + {import.meta.env.INSTRUMENTATION_ENABLED === "true" && ( + <> + + + + Settings + + + + + + + Templates + + + + + )} diff --git a/ui/vite.config.ts b/ui/vite.config.ts index 091e15f36..39bafbade 100644 --- a/ui/vite.config.ts +++ b/ui/vite.config.ts @@ -22,6 +22,9 @@ export default defineConfig(({ mode }) => { "import.meta.env.VITE_SERVER_URL": JSON.stringify( env.UI_SERVER_URL || "http://localhost:3000", ), + "import.meta.env.INSTRUMENTATION_ENABLED": JSON.stringify( + env.INSTRUMENTATION_ENABLED || "false", + ), }, build: { outDir: "dist",