Monitoring AI Agents in Production: What You Actually Need

// lib/ai-telemetry.ts
export interface AgentTelemetry {
  // Identity
  traceId: string;
  sessionId: string;
  agentId: string;
  userId?: string;

  // Input
  inputTokens: number;
  inputLength: number;
  systemPromptLength: number;
  contextDocuments: number; // For RAG agents

  // Processing
  model: string;
  temperature: number;
  latencyMs: number;
  retryCount: number;

  // Output
  outputTokens: number;
  outputLength: number;
  finishReason: "end_turn" | "max_tokens" | "stop_sequence" | "tool_use";
  toolsUsed: string[];
  toolCallCount: number;

  // Cost
  inputCostUsd: number;
  outputCostUsd: number;
  totalCostUsd: number;

  // Quality signals
  userFeedback?: "positive" | "negative";
  escalated: boolean;
  errorCode?: string;
  errorMessage?: string;

  // Timestamps
  timestamp: number;
}

// Cost calculation per model
const COST_PER_MILLION_TOKENS = {
  "claude-haiku-4-20250514": { input: 0.25, output: 1.25 },
  "claude-sonnet-4-20250514": { input: 3.0, output: 15.0 },
  "claude-opus-4-20250514": { input: 15.0, output: 75.0 },
} as const;

export function calculateCost(
  model: string,
  inputTokens: number,
  outputTokens: number
): { inputCostUsd: number; outputCostUsd: number; totalCostUsd: number } {
  const costs = COST_PER_MILLION_TOKENS[model as keyof typeof COST_PER_MILLION_TOKENS] ??
    { input: 3.0, output: 15.0 }; // Default to Sonnet pricing

  const inputCostUsd = (inputTokens / 1_000_000) * costs.input;
  const outputCostUsd = (outputTokens / 1_000_000) * costs.output;

  return {
    inputCostUsd,
    outputCostUsd,
    totalCostUsd: inputCostUsd + outputCostUsd,
  };
}

// lib/monitored-agent.ts
import Anthropic from "@anthropic-ai/sdk";
import { nanoid } from "nanoid";
import { calculateCost, AgentTelemetry } from "./ai-telemetry.js";

const anthropic = new Anthropic();

interface CreateMessageOptions {
  model: string;
  system?: string;
  messages: Anthropic.MessageParam[];
  max_tokens: number;
  temperature?: number;
  tools?: Anthropic.Tool[];
  agentId: string;
  sessionId: string;
  userId?: string;
}

export async function createMonitoredMessage(
  options: CreateMessageOptions,
  onTelemetry: (telemetry: AgentTelemetry) => Promise<void>
): Promise<Anthropic.Message> {
  const traceId = nanoid();
  const startTime = Date.now();
  let retryCount = 0;
  let lastError: Error | null = null;

  const { agentId, sessionId, userId, ...claudeOptions } = options;

  while (retryCount < 3) {
    try {
      const response = await anthropic.messages.create(claudeOptions);
      const latencyMs = Date.now() - startTime;

      const costs = calculateCost(
        claudeOptions.model,
        response.usage.input_tokens,
        response.usage.output_tokens
      );

      const toolsUsed: string[] = [];
      let toolCallCount = 0;

      for (const block of response.content) {
        if (block.type === "tool_use") {
          toolsUsed.push(block.name);
          toolCallCount++;
        }
      }

      const telemetry: AgentTelemetry = {
        traceId,
        sessionId,
        agentId,
        userId,
        inputTokens: response.usage.input_tokens,
        inputLength: claudeOptions.messages.reduce(
          (sum, m) => sum + (typeof m.content === "string" ? m.content.length : 0), 0
        ),
        systemPromptLength: claudeOptions.system?.length ?? 0,
        contextDocuments: 0,
        model: claudeOptions.model,
        temperature: claudeOptions.temperature ?? 1,
        latencyMs,
        retryCount,
        outputTokens: response.usage.output_tokens,
        outputLength: response.content
          .filter(b => b.type === "text")
          .reduce((sum, b) => sum + (b as Anthropic.TextBlock).text.length, 0),
        finishReason: response.stop_reason as AgentTelemetry["finishReason"],
        toolsUsed,
        toolCallCount,
        ...costs,
        escalated: false,
        timestamp: Date.now(),
      };

      await onTelemetry(telemetry);

      return response;
    } catch (error) {
      lastError = error as Error;

      // Don't retry on non-transient errors
      if (error instanceof Anthropic.APIError && error.status === 400) {
        break;
      }

      retryCount++;
      if (retryCount < 3) {
        await new Promise(resolve => setTimeout(resolve, 1000 * retryCount));
      }
    }
  }

  // Log the failed attempt
  await onTelemetry({
    traceId,
    sessionId,
    agentId,
    userId,
    inputTokens: 0,
    inputLength: 0,
    systemPromptLength: 0,
    contextDocuments: 0,
    model: claudeOptions.model,
    temperature: claudeOptions.temperature ?? 1,
    latencyMs: Date.now() - startTime,
    retryCount,
    outputTokens: 0,
    outputLength: 0,
    finishReason: "end_turn",
    toolsUsed: [],
    toolCallCount: 0,
    inputCostUsd: 0,
    outputCostUsd: 0,
    totalCostUsd: 0,
    escalated: false,
    errorCode: lastError?.name,
    errorMessage: lastError?.message,
    timestamp: Date.now(),
  });

  throw lastError;
}

// lib/cost-guard.ts
export interface CostLimits {
  perSessionUsd: number;    // Max cost per conversation
  perUserDayUsd: number;    // Max daily cost per user
  globalHourUsd: number;    // Max hourly total cost
}

const DEFAULT_LIMITS: CostLimits = {
  perSessionUsd: 0.50,      // $0.50 per conversation max
  perUserDayUsd: 2.00,      // $2.00 per user per day
  globalHourUsd: 50.00,     // $50 per hour total
};

export class CostGuard {
  private sessionCosts = new Map<string, number>();
  private userDayCosts = new Map<string, number>();
  private globalHourlyCost = 0;
  private hourWindowStart = Date.now();

  check(
    sessionId: string,
    userId: string | undefined,
    cost: number,
    limits: CostLimits = DEFAULT_LIMITS
  ): { allowed: boolean; reason?: string } {
    // Reset hourly window
    if (Date.now() - this.hourWindowStart > 3600000) {
      this.globalHourlyCost = 0;
      this.hourWindowStart = Date.now();
    }

    const sessionCost = (this.sessionCosts.get(sessionId) ?? 0) + cost;
    if (sessionCost > limits.perSessionUsd) {
      return { allowed: false, reason: "Session cost limit exceeded" };
    }

    if (userId) {
      const userKey = `${userId}:${new Date().toDateString()}`;
      const userCost = (this.userDayCosts.get(userKey) ?? 0) + cost;
      if (userCost > limits.perUserDayUsd) {
        return { allowed: false, reason: "Daily user cost limit exceeded" };
      }
      this.userDayCosts.set(userKey, userCost);
    }

    if (this.globalHourlyCost + cost > limits.globalHourUsd) {
      return { allowed: false, reason: "Global hourly cost limit exceeded" };
    }

    // Update costs
    this.sessionCosts.set(sessionId, sessionCost);
    this.globalHourlyCost += cost;

    return { allowed: true };
  }
}

export const costGuard = new CostGuard();

// lib/quality-monitor.ts
import Anthropic from "@anthropic-ai/sdk";

export interface QualityCheck {
  passed: boolean;
  issues: string[];
  score: number; // 0-1
}

const anthropic = new Anthropic();

export async function checkResponseQuality(
  userMessage: string,
  agentResponse: string,
  context: {
    agentPurpose: string;
    expectedTopics: string[];
  }
): Promise<QualityCheck> {
  const response = await anthropic.messages.create({
    model: "claude-haiku-4-20250514", // Use cheaper model for monitoring
    max_tokens: 512,
    system: `You are a quality evaluator for AI agent responses. Return JSON only.`,
    messages: [
      {
        role: "user",
        content: `Evaluate this AI response for quality issues.

Agent purpose: ${context.agentPurpose}
Expected topics: ${context.expectedTopics.join(", ")}

User message: ${userMessage}
Agent response: ${agentResponse}

Check for:
1. Is the response on-topic?
2. Does it contain hallucinated facts?
3. Is it appropriate (no harmful content)?
4. Does it answer the user's actual question?
5. Is it coherent?

Return: { passed: boolean, issues: string[], score: 0-1 }`,
      },
    ],
  });

  const text = response.content[0].type === "text" ? response.content[0].text : "{}";

  try {
    return JSON.parse(text);
  } catch {
    return { passed: true, issues: [], score: 0.8 }; // Default to passing
  }
}

// Save telemetry to a file or simple database
import * as fs from "fs/promises";

export async function logTelemetry(telemetry: AgentTelemetry): Promise<void> {
  // Append to JSONL file (one JSON object per line)
  await fs.appendFile(
    "./logs/agent-telemetry.jsonl",
    JSON.stringify(telemetry) + "\n"
  );

  // Alert on cost anomalies
  if (telemetry.totalCostUsd > 0.10) {
    console.warn(`High cost session: $${telemetry.totalCostUsd.toFixed(4)} for ${telemetry.sessionId}`);
    // Add Slack/email notification here
  }

  // Alert on slow responses
  if (telemetry.latencyMs > 10000) {
    console.warn(`Slow response: ${telemetry.latencyMs}ms for ${telemetry.traceId}`);
  }
}