# Ключевые метрики AI-агента
📊 Бизнес-метрики
  — requests_total: общее количество запросов
  — success_rate: доля успешных ответов
  — task_completion_rate: процент выполненных задач

⚡ Технические метрики
  — llm_latency_seconds: время ответа LLM
  — tokens_used: потрачено токенов
  — tool_call_count: количество вызовов инструментов

💰 Стоимость
  — cost_per_request: $ за запрос
  — cost_per_day: $ в день

🚨 Алерты
  — error_rate > 5% → оповещение
  — llm_latency > 30s → оповещение
  — cost_per_day > бюджет → оповещение

from prometheus_client import Counter, Histogram, Gauge, generate_latest
from fastapi import FastAPI, Response
import time

app = FastAPI()

# Метрики
REQUESTS = Counter("agent_requests_total", "Всего запросов", ["status"])
TOKENS = Counter("agent_tokens_total", "Потрачено токенов", ["type"])
LATENCY = Histogram("agent_llm_latency_seconds", "Latency LLM", buckets=[0.5, 1, 3, 5, 10, 30, 60])
TOOLS = Counter("agent_tool_calls_total", "Вызовов инструментов", ["tool"])
COST = Counter("agent_cost_cents_total", "Стоимость в центах", ["model"])

ACTIVE_REQUESTS = Gauge("agent_active_requests", "Активных запросов сейчас")

# Эндпоинт для Prometheus
@app.get("/metrics")
def metrics():
    return Response(content=generate_latest(), media_type="text/plain")

# Декоратор для замера latency
def track_llm_call(func):
    def wrapper(*args, **kwargs):
        ACTIVE_REQUESTS.inc()
        start = time.time()
        try:
            result = func(*args, **kwargs)
            REQUESTS.labels(status="success").inc()
            return result
        except Exception:
            REQUESTS.labels(status="error").inc()
            raise
        finally:
            elapsed = time.time() - start
            LATENCY.observe(elapsed)
            ACTIVE_REQUESTS.dec()
    return wrapper

# prometheus.yml
global:
  scrape_interval: 15s
  evaluation_interval: 15s

alerting:
  alertmanagers:
    - static_configs:
        - targets: ["alertmanager:9093"]

rule_files:
  - "alerts.yml"

scrape_configs:
  - job_name: "ai-agent"
    static_configs:
      - targets: ["agent:8000"]
    metrics_path: "/metrics"

# alerts.yml — правила алертов
groups:
  - name: ai_agent_alerts
    rules:
      - alert: HighErrorRate
        expr: |
          rate(agent_requests_total{status="error"}[5m])
          / rate(agent_requests_total[5m]) > 0.05
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "Error rate > 5% ({{ $value | humanizePercentage }})"

      - alert: HighLLMLatency
        expr: |
          histogram_quantile(0.95, rate(agent_llm_latency_seconds_bucket[5m])) > 30
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "P95 LLM latency > 30s"

      - alert: BudgetExceeded
        expr: increase(agent_cost_cents_total[24h]) > 1000
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "Дневной бюджет превышен: ${{ $value | humanize }} > $10"

# alertmanager.yml — роутинг в Telegram
route:
  receiver: "telegram"
  group_by: ["alertname"]

receivers:
  - name: "telegram"
    telegram_configs:
      - bot_token: "YOUR_BOT_TOKEN"
        chat_id: -1001234567890
        message: "🚨 {{ .GroupLabels.alertname }}\n{{ .CommonAnnotations.summary }}"

# docker-compose.yml — весь стек мониторинга
version: "3.8"
services:
  agent:
    build: .
    ports: ["8000:8000"]
    environment:
      OPENAI_API_KEY: ${OPENAI_API_KEY}

  prometheus:
    image: prom/prometheus:v3.2.0
    ports: ["9090:9090"]
    volumes:
      - ./prometheus.yml:/etc/prometheus/prometheus.yml
      - ./alerts.yml:/etc/prometheus/alerts.yml
      - prometheus_data:/prometheus
    command: --storage.tsdb.retention.time=30d

  grafana:
    image: grafana/grafana:11.5.0
    ports: ["3000:3000"]
    environment:
      GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_PASSWORD:-admin}
    volumes:
      - grafana_data:/var/lib/grafana

  # AlertManager для уведомлений
  alertmanager:
    image: prom/alertmanager:v0.28.0
    ports: ["9093:9093"]
    volumes:
      - ./alertmanager.yml:/etc/alertmanager/alertmanager.yml

volumes:
  prometheus_data:
  grafana_data:

from opentelemetry import trace
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
from opentelemetry.sdk.trace.export import BatchSpanProcessor

# Настройка трейсинга
provider = TracerProvider()
exporter = OTLPSpanExporter(endpoint="http://jaeger:4317")
provider.add_span_processor(BatchSpanProcessor(exporter))
trace.set_tracer_provider(provider)

tracer = trace.get_tracer("ai-agent")

def process_user_request(user_input):
    with tracer.start_as_current_span("agent.request") as span:
        span.set_attribute("user.input.length", len(user_input))
        
        # LLM-вызов
        with tracer.start_as_current_span("llm.call") as llm_span:
            response = client.chat.completions.create(
                model="gpt-4o", messages=[{"role": "user", "content": user_input}]
            )
            llm_span.set_attribute("tokens.used", response.usage.total_tokens)
        
        # Если tool call — трейсим и его
        if response.choices[0].finish_reason == "tool_calls":
            with tracer.start_as_current_span("tool.execute") as tool_span:
                tool_call = response.choices[0].message.tool_calls[0]
                tool_span.set_attribute("tool.name", tool_call.function.name)
                # execute tool...
        
        return response