AI Observability
& OpenTelemetry.

# OTel Python SDK ve OTLP exporter
pip install opentelemetry-sdk \
            opentelemetry-exporter-otlp \
            opentelemetry-instrumentation-fastapi \
            opentelemetry-instrumentation-httpx

from opentelemetry import trace
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
from opentelemetry.sdk.resources import Resource

def setup_tracer(service_name: str) -> trace.Tracer:
    resource = Resource(attributes={"service.name": service_name})

    provider = TracerProvider(resource=resource)

    # OTLP gRPC ile Jaeger / Grafana Tempo'ya gönder
    exporter = OTLPSpanExporter(endpoint="http://localhost:4317")
    provider.add_span_processor(BatchSpanProcessor(exporter))

    trace.set_tracer_provider(provider)
    return trace.get_tracer(service_name)

tracer = setup_tracer("llm-service")

import time
from opentelemetry import trace
from openai import OpenAI

tracer = trace.get_tracer("llm-service")
client = OpenAI()

def traced_chat(
    messages: list[dict],
    model: str = "gpt-4o",
    temperature: float = 0.7,
) -> str:
    with tracer.start_as_current_span("llm.chat_completion") as span:
        span.set_attribute("llm.model", model)
        span.set_attribute("llm.temperature", temperature)
        span.set_attribute("llm.message_count", len(messages))

        t0 = time.perf_counter()
        try:
            response = client.chat.completions.create(
                model=model,
                messages=messages,
                temperature=temperature,
            )
            latency_ms = (time.perf_counter() - t0) * 1000

            usage = response.usage
            span.set_attribute("llm.prompt_tokens", usage.prompt_tokens)
            span.set_attribute("llm.completion_tokens", usage.completion_tokens)
            span.set_attribute("llm.total_tokens", usage.total_tokens)
            span.set_attribute("llm.latency_ms", round(latency_ms, 2))
            span.set_attribute("llm.finish_reason", response.choices[0].finish_reason)

            span.set_status(trace.Status(trace.StatusCode.OK))
            return response.choices[0].message.content

        except Exception as e:
            span.set_status(trace.Status(trace.StatusCode.ERROR, str(e)))
            span.record_exception(e)
            raise

from opentelemetry import trace
from llm_tracer import traced_chat

tracer = trace.get_tracer("llm-service")

def rag_query(question: str, vector_db, reranker) -> str:
    with tracer.start_as_current_span("chain.run") as chain_span:
        chain_span.set_attribute("chain.question", question[:200])

        # Retrieval span
        with tracer.start_as_current_span("retrieval.search") as ret_span:
            chunks = vector_db.similarity_search(question, k=10)
            ret_span.set_attribute("retrieval.num_results", len(chunks))

        # Reranking span
        with tracer.start_as_current_span("reranker.score") as rer_span:
            top_chunks = reranker.rerank(question, chunks, top_n=3)
            rer_span.set_attribute("reranker.top_n", len(top_chunks))

        context = "\n\n".join([c.page_content for c in top_chunks])
        messages = [
            {"role": "system", "content": "Yalnızca verilen bağlamı kullan."},
            {"role": "user", "content": f"Bağlam:\n{context}\n\nSoru: {question}"},
        ]

        return traced_chat(messages)

import tiktoken
from dataclasses import dataclass
from opentelemetry import metrics

# Metrik meter (Prometheus uyumlu)
meter = metrics.get_meter("llm-cost-tracker")
cost_counter = meter.create_counter(
    "llm_cost_usd_total",
    description="Toplam LLM maliyeti (USD)",
    unit="USD",
)
token_histogram = meter.create_histogram(
    "llm_tokens_per_request",
    description="İstek başına token dağılımı",
)

# Model başına fiyat tablosu ($/1M token)
PRICE_TABLE: dict[str, dict] = {
    "gpt-4o":          {"prompt": 2.50,  "completion": 10.00},
    "gpt-4o-mini":     {"prompt": 0.15,  "completion": 0.60},
    "gpt-4-turbo":     {"prompt": 10.00, "completion": 30.00},
    "gpt-3.5-turbo":   {"prompt": 0.50,  "completion": 1.50},
}

@dataclass
class CostResult:
    prompt_tokens: int
    completion_tokens: int
    total_tokens: int
    cost_usd: float

def count_tokens(text: str, model: str = "gpt-4o") -> int:
    """API çağrısı yapmadan token sayısını hesapla."""
    try:
        enc = tiktoken.encoding_for_model(model)
    except KeyError:
        enc = tiktoken.get_encoding("cl100k_base")
    return len(enc.encode(text))

def calculate_cost(
    prompt_tokens: int,
    completion_tokens: int,
    model: str,
) -> CostResult:
    prices = PRICE_TABLE.get(model, {"prompt": 0, "completion": 0})
    cost = (
        (prompt_tokens     / 1_000_000) * prices["prompt"]
      + (completion_tokens / 1_000_000) * prices["completion"]
    )
    total = prompt_tokens + completion_tokens

    # OTel metrik kaydet
    labels = {"model": model}
    cost_counter.add(cost, labels)
    token_histogram.record(total, labels)

    return CostResult(
        prompt_tokens=prompt_tokens,
        completion_tokens=completion_tokens,
        total_tokens=total,
        cost_usd=round(cost, 8),
    )

# Kullanım örneği
if __name__ == "__main__":
    prompt = "Python'da bir async web scraper nasıl yazılır?"
    n = count_tokens(prompt)
    print(f"Prompt token sayısı: {n}")

    result = calculate_cost(n, 250, "gpt-4o")
    print(f"Tahmini maliyet: ${result.cost_usd:.6f}")

pip install langfuse openai

# .env dosyasına ekle
LANGFUSE_PUBLIC_KEY=pk-lf-...
LANGFUSE_SECRET_KEY=sk-lf-...
LANGFUSE_HOST=https://cloud.langfuse.com  # veya self-hosted URL

from langfuse import Langfuse
from langfuse.decorators import langfuse_context, observe
from openai import OpenAI
import os

lf = Langfuse(
    public_key=os.environ["LANGFUSE_PUBLIC_KEY"],
    secret_key=os.environ["LANGFUSE_SECRET_KEY"],
    host=os.environ.get("LANGFUSE_HOST", "https://cloud.langfuse.com"),
)
client = OpenAI()

# @observe dekoratörü ile otomatik trace
@observe()
def answer_question(question: str, user_id: str) -> str:
    # Prompt versiyonunu Langfuse'dan çek
    prompt_obj = lf.get_prompt("rag-system-prompt", label="production")
    system_msg = prompt_obj.compile(language="tr")

    langfuse_context.update_current_trace(
        name="answer_question",
        user_id=user_id,
        tags=["rag", "production"],
    )

    response = client.chat.completions.create(
        model="gpt-4o",
        messages=[
            {"role": "system", "content": system_msg},
            {"role": "user", "content": question},
        ],
    )

    answer = response.choices[0].message.content

    # Kalite skoru kaydet
    langfuse_context.score_current_trace(
        name="relevance",
        value=1.0,  # kullanıcı geri bildirimi ile güncelle
        comment="auto-score",
    )
    return answer

# Yeni prompt versiyonu oluştur
lf.create_prompt(
    name="rag-system-prompt",
    prompt="""Sen  dilinde yanıt veren yardımcı bir asistansın.
Yalnızca aşağıdaki bağlam bilgisini kullan.
Bağlamda yoksa 'Bilmiyorum' de.

Bağlam:
""",
    labels=["staging"],  # önce staging'e gönder
    config={"model": "gpt-4o", "temperature": 0.3},
)

# A/B test için ikinci versiyon
lf.create_prompt(
    name="rag-system-prompt",
    prompt="""Görevin:  dilinde kısa ve öz yanıtlar vermek.
Sadece belirtilen kaynakları kullan. Kaynak yoksa reddet.

Kaynaklar:
""",
    labels=["experiment-v2"],
)

from prometheus_client import (
    Histogram, Counter, Gauge, start_http_server
)
import time, threading

# Histogram: latency dağılımı (bucket sınırları ms cinsinden)
LLM_LATENCY = Histogram(
    "llm_request_latency_ms",
    "LLM isteği latency dağılımı",
    labelnames=["model", "endpoint"],
    buckets=[50, 100, 250, 500, 1000, 2000, 5000, 10000],
)
TTFT_HISTOGRAM = Histogram(
    "llm_ttft_ms",
    "Time-to-first-token (ms)",
    labelnames=["model"],
    buckets=[100, 250, 500, 750, 1000, 1500, 2000],
)
TOKEN_THROUGHPUT = Gauge(
    "llm_tokens_per_second",
    "Anlık token throughput",
    labelnames=["model"],
)
REQUEST_COUNTER = Counter(
    "llm_requests_total",
    "Toplam LLM istek sayısı",
    labelnames=["model", "status"],
)

def record_latency(model: str, endpoint: str, latency_ms: float, status: str):
    LLM_LATENCY.labels(model=model, endpoint=endpoint).observe(latency_ms)
    REQUEST_COUNTER.labels(model=model, status=status).inc()

def record_streaming_metrics(model: str, ttft_ms: float, total_tokens: int, total_ms: float):
    TTFT_HISTOGRAM.labels(model=model).observe(ttft_ms)
    if total_ms > 0:
        throughput = (total_tokens / total_ms) * 1000  # tokens/s
        TOKEN_THROUGHPUT.labels(model=model).set(throughput)

# Prometheus /metrics endpoint'i başlat
def start_metrics_server(port: int = 8001):
    threading.Thread(
        target=start_http_server,
        args=(port,),
        daemon=True,
    ).start()
    print(f"Prometheus metrics: http://localhost:{port}/metrics")

import numpy as np
from scipy import stats
from scipy.spatial.distance import jensenshannon
from openai import OpenAI
from dataclasses import dataclass, field
import json, pathlib

client = OpenAI()

@dataclass
class DriftReport:
    ks_statistic: float
    ks_pvalue: float
    js_divergence: float
    is_drifted: bool
    alert_message: str = ""

def embed_texts(texts: list[str]) -> np.ndarray:
    response = client.embeddings.create(
        input=texts,
        model="text-embedding-3-small",
    )
    return np.array([e.embedding for e in response.data])

def detect_embedding_drift(
    baseline_texts: list[str],
    current_texts: list[str],
    ks_alpha: float = 0.05,
    js_threshold: float = 0.1,
) -> DriftReport:
    """Baseline ve güncel embedding'ler arasında drift tespit et."""

    baseline_emb = embed_texts(baseline_texts)
    current_emb  = embed_texts(current_texts)

    # PCA ile boyut indir, ilk komponentin dağılımını karşılaştır
    from sklearn.decomposition import PCA
    pca = PCA(n_components=1)
    baseline_proj = pca.fit_transform(baseline_emb).flatten()
    current_proj  = pca.transform(current_emb).flatten()

    # KS testi: dağılımlar aynı mı?
    ks_stat, ks_p = stats.ks_2samp(baseline_proj, current_proj)

    # Jensen-Shannon divergence (histogram karşılaştırması)
    bins = np.linspace(
        min(baseline_proj.min(), current_proj.min()),
        max(baseline_proj.max(), current_proj.max()),
        30,
    )
    hist_a, _ = np.histogram(baseline_proj, bins=bins, density=True)
    hist_b, _ = np.histogram(current_proj,  bins=bins, density=True)

    # Sıfırları önle
    hist_a = hist_a + 1e-10
    hist_b = hist_b + 1e-10
    js_div = float(jensenshannon(hist_a, hist_b))

    is_drifted = (ks_p < ks_alpha) or (js_div > js_threshold)
    msg = ""
    if is_drifted:
        msg = (f"DRIFT UYARISI: KS={ks_stat:.3f} (p={ks_p:.4f}), "
               f"JS={js_div:.3f}. Prompt şablonunu güncelle.")

    return DriftReport(
        ks_statistic=round(ks_stat, 4),
        ks_pvalue=round(ks_p, 4),
        js_divergence=round(js_div, 4),
        is_drifted=is_drifted,
        alert_message=msg,
    )

# Kullanım örneği
if __name__ == "__main__":
    baseline = [
        "Python nedir?", "Liste nasıl oluşturulur?",
        "Fonksiyon tanımla", "Döngü örneği ver",
    ]
    current = [
        "Kubernetes pod nedir?", "Helm chart nasıl yazılır?",
        "Ingress controller kur", "Service mesh nedir?",
    ]
    report = detect_embedding_drift(baseline, current)
    print(report)

import httpx, json
from datetime import datetime
from enum import Enum

class Severity(Enum):
    INFO     = "#36a64f"  # yeşil
    WARNING  = "#ffa500"  # turuncu
    CRITICAL = "#ff0000"  # kırmızı

async def send_slack_alert(
    webhook_url: str,
    title: str,
    message: str,
    severity: Severity = Severity.WARNING,
    fields: dict | None = None,
):
    payload = {
        "attachments": [{
            "color": severity.value,
            "title": f"[LLM Alert] {title}",
            "text": message,
            "fields": [
                {"title": k, "value": str(v), "short": True}
                for k, v in (fields or {}).items()
            ],
            "footer": "LLM Observability",
            "ts": int(datetime.now().timestamp()),
        }]
    }
    async with httpx.AsyncClient() as c:
        resp = await c.post(webhook_url, json=payload)
        resp.raise_for_status()

# Alert örneği: yüksek latency
async def check_latency_sla(p95_ms: float, webhook_url: str):
    if p95_ms > 5000:
        await send_slack_alert(
            webhook_url,
            title="Yüksek Latency",
            message=f"P95 latency SLA ihlali: {p95_ms:.0f}ms (limit: 5000ms)",
            severity=Severity.CRITICAL,
            fields={"P95 Latency": f"{p95_ms:.0f}ms", "SLA Limit": "5000ms"},
        )
    elif p95_ms > 2000:
        await send_slack_alert(
            webhook_url,
            title="Latency Uyarısı",
            message=f"P95 latency yüksek: {p95_ms:.0f}ms",
            severity=Severity.WARNING,
        )

{
  "title": "LLM Observability",
  "uid": "llm-obs-v1",
  "panels": [
    {
      "title": "P95 Latency (ms)",
      "type": "timeseries",
      "targets": [{
        "expr": "histogram_quantile(0.95, rate(llm_request_latency_ms_bucket[5m]))",
        "legendFormat": ""
      }],
      "alert": {
        "conditions": [{
          "evaluator": {"params": [2000], "type": "gt"},
          "type": "query"
        }]
      }
    },
    {
      "title": "Saatlik Token Maliyeti (USD)",
      "type": "stat",
      "targets": [{
        "expr": "increase(llm_cost_usd_total[1h])"
      }]
    },
    {
      "title": "Hata Oranı (%)",
      "type": "timeseries",
      "targets": [{
        "expr": "100 * rate(llm_requests_total{status='error'}[5m]) / rate(llm_requests_total[5m])"
      }]
    },
    {
      "title": "Token Throughput (tokens/s)",
      "type": "gauge",
      "targets": [{
        "expr": "llm_tokens_per_second"
      }]
    }
  ]
}

version: "3.9"
services:
  prometheus:
    image: prom/prometheus:v2.51.0
    volumes:
      - ./prometheus.yml:/etc/prometheus/prometheus.yml
    ports: ["9090:9090"]

  grafana:
    image: grafana/grafana:10.4.0
    environment:
      - GF_SECURITY_ADMIN_PASSWORD=admin
    volumes:
      - ./grafana/dashboards:/etc/grafana/provisioning/dashboards
    ports: ["3000:3000"]
    depends_on: [prometheus]

  otel-collector:
    image: otel/opentelemetry-collector-contrib:0.98.0
    volumes:
      - ./otel-config.yml:/etc/otelcol-contrib/config.yaml
    ports:
      - "4317:4317"   # gRPC
      - "4318:4318"   # HTTP

  langfuse:
    image: langfuse/langfuse:2
    environment:
      - DATABASE_URL=postgresql://langfuse:langfuse@postgres/langfuse
      - NEXTAUTH_SECRET=secret123
      - SALT=salt123
    ports: ["3010:3000"]
    depends_on: [postgres]

  postgres:
    image: postgres:16-alpine
    environment:
      - POSTGRES_USER=langfuse
      - POSTGRES_PASSWORD=langfuse
      - POSTGRES_DB=langfuse