vLLM &
TensorRT-LLM.

# CUDA 12.1+ gerektirir
pip install vllm

# Belirli CUDA versiyonu için
pip install vllm --extra-index-url https://download.pytorch.org/whl/cu121

# Docker ile (önerilen production yöntemi)
docker pull vllm/vllm-openai:latest

from vllm import LLM, SamplingParams

# Modeli yükle
llm = LLM(
    model="meta-llama/Meta-Llama-3-8B-Instruct",
    tensor_parallel_size=1,   # tek GPU
    gpu_memory_utilization=0.90,
    max_model_len=4096,
    dtype="bfloat16",
)

# Sampling parametreleri
sampling_params = SamplingParams(
    temperature=0.7,
    top_p=0.95,
    max_tokens=512,
    stop=["<|eot_id|>"],
)

# Batch inference — tüm prompts tek seferde işlenir
prompts = [
    "Transformer mimarisini açıkla:",
    "Python'da async/await nedir?",
    "Docker container neden kullanılır?",
]

outputs = llm.generate(prompts, sampling_params)

for output in outputs:
    prompt = output.prompt
    generated_text = output.outputs[0].text
    print(f"Prompt: {prompt!r}")
    print(f"Output: {generated_text!r}")
    print("-" * 60)

from vllm import LLM, SamplingParams

llm = LLM(model="meta-llama/Meta-Llama-3-8B-Instruct")

# Chat formatında mesajlar
conversation = [
    {"role": "system", "content": "Sen yardımcı bir asistansın."},
    {"role": "user",   "content": "vLLM neden bu kadar hızlı?"},
]

# Tokenizer'ın chat template'ini kullan
tokenizer = llm.get_tokenizer()
prompt = tokenizer.apply_chat_template(
    conversation,
    tokenize=False,
    add_generation_prompt=True,
)

outputs = llm.generate([prompt], SamplingParams(max_tokens=256))
print(outputs[0].outputs[0].text)

# Sunucuyu başlat
python -m vllm.entrypoints.openai.api_server \
  --model meta-llama/Meta-Llama-3-8B-Instruct \
  --host 0.0.0.0 \
  --port 8000 \
  --tensor-parallel-size 1 \
  --gpu-memory-utilization 0.90 \
  --max-model-len 8192 \
  --served-model-name llama-3-8b

from openai import OpenAI

# vLLM sunucusuna yönlendir
client = OpenAI(
    base_url="http://localhost:8000/v1",
    api_key="token-abc123",  # herhangi bir değer
)

# Chat completion
response = client.chat.completions.create(
    model="llama-3-8b",
    messages=[
        {"role": "system", "content": "Kısa ve net yanıt ver."},
        {"role": "user",   "content": "PagedAttention nedir?"},
    ],
    max_tokens=256,
    temperature=0.7,
    stream=False,
)
print(response.choices[0].message.content)

import sys

stream = client.chat.completions.create(
    model="llama-3-8b",
    messages=[{"role": "user", "content": "Bana bir hikaye anlat."}],
    max_tokens=512,
    stream=True,
)

for chunk in stream:
    delta = chunk.choices[0].delta
    if delta.content:
        sys.stdout.write(delta.content)
        sys.stdout.flush()
print()  # newline

# 4 GPU için tensor parallelism
python -m vllm.entrypoints.openai.api_server \
  --model meta-llama/Meta-Llama-3-70B-Instruct \
  --tensor-parallel-size 4 \
  --pipeline-parallel-size 1 \
  --gpu-memory-utilization 0.95 \
  --max-model-len 8192 \
  --dtype bfloat16

# 8 GPU: tensor × pipeline parallelism
python -m vllm.entrypoints.openai.api_server \
  --model meta-llama/Meta-Llama-3-70B-Instruct \
  --tensor-parallel-size 4 \
  --pipeline-parallel-size 2 \
  --max-model-len 4096

from vllm import LLM

# 4 GPU'ya tensor parallelism
llm = LLM(
    model="meta-llama/Meta-Llama-3-70B-Instruct",
    tensor_parallel_size=4,
    gpu_memory_utilization=0.95,
    dtype="bfloat16",
    enforce_eager=False,   # CUDA graph kullan (daha hızlı)
)

# GPU kullanımını kontrol et
import torch
for i in range(torch.cuda.device_count()):
    mem = torch.cuda.memory_allocated(i) / 1024**3
    print(f"GPU {i}: {mem:.1f} GB kullanımda")

# Eagle draft model ile speculative decoding
python -m vllm.entrypoints.openai.api_server \
  --model meta-llama/Meta-Llama-3-8B-Instruct \
  --speculative-model eagle \
  --num-speculative-tokens 5 \
  --speculative-draft-tensor-parallel-size 1

# Küçük model ile (harici draft)
python -m vllm.entrypoints.openai.api_server \
  --model meta-llama/Meta-Llama-3-70B-Instruct \
  --speculative-model meta-llama/Meta-Llama-3-8B-Instruct \
  --num-speculative-tokens 5 \
  --tensor-parallel-size 4

from vllm import LLM, SamplingParams

# Medusa: hedef modele eklenen ek head'ler
llm = LLM(
    model="FasterDecoding/medusa-vicuna-7b-v1.3",
    speculative_model="[medusa]",
    num_speculative_tokens=3,
    tensor_parallel_size=1,
)

outputs = llm.generate(
    ["Python'da decorator nasıl kullanılır?"],
    SamplingParams(max_tokens=256, temperature=0.0),
)
print(outputs[0].outputs[0].text)

# Docker önerilen yöntem
docker pull nvcr.io/nvidia/tritonserver:24.01-trtllm-python-py3

# pip install (CUDA 12.x)
pip install tensorrt-llm -U --extra-index-url https://pypi.nvidia.com

# Model derleme (Llama-3 örneği)
python examples/llama/convert_checkpoint.py \
  --model_dir /models/llama-3-8b-hf \
  --output_dir /models/llama-3-8b-trt-ckpt \
  --dtype bfloat16 \
  --tp_size 1

trtllm-build \
  --checkpoint_dir /models/llama-3-8b-trt-ckpt \
  --output_dir /models/llama-3-8b-trt-engine \
  --gemm_plugin bfloat16 \
  --max_batch_size 32 \
  --max_input_len 2048 \
  --max_seq_len 4096

import tensorrt_llm
from tensorrt_llm.runtime import ModelRunnerCpp

# Engine'i yükle
runner = ModelRunnerCpp.from_dir(
    engine_dir="/models/llama-3-8b-trt-engine",
    rank=tensorrt_llm.mpi_rank(),
)

# Batch inference
import torch
input_ids = [
    torch.tensor([1, 29871, 13, 450], dtype=torch.int32),
    torch.tensor([1, 29871, 13], dtype=torch.int32),
]

outputs = runner.generate(
    batch_input_ids=input_ids,
    max_new_tokens=256,
    temperature=0.7,
    end_id=2,
    pad_id=2,
)
print(outputs)

# AWQ (Activation-Aware Weight Quantization) — önerilen INT4
pip install autoawq

# Hugging Face'den hazır AWQ model
python -m vllm.entrypoints.openai.api_server \
  --model TheBloke/Llama-3-8B-Instruct-AWQ \
  --quantization awq \
  --dtype half

# GPTQ model
python -m vllm.entrypoints.openai.api_server \
  --model TheBloke/Llama-3-8B-Instruct-GPTQ \
  --quantization gptq \
  --dtype half

from awq import AutoAWQForCausalLM
from transformers import AutoTokenizer

model_path = "meta-llama/Meta-Llama-3-8B-Instruct"
quant_path = "llama-3-8b-awq"

# Quantization konfigürasyonu
quant_config = {
    "zero_point": True,
    "q_group_size": 128,
    "w_bit": 4,           # INT4
    "version": "GEMM",
}

# Model ve tokenizer yükle
model = AutoAWQForCausalLM.from_pretrained(model_path)
tokenizer = AutoTokenizer.from_pretrained(model_path)

# Kalibrasyon verisi ile quantize et
model.quantize(tokenizer, quant_config=quant_config)
model.save_quantized(quant_path)
tokenizer.save_pretrained(quant_path)
print(f"AWQ model kaydedildi: {quant_path}")

# vLLM'in dahili benchmark aracı
python benchmarks/benchmark_throughput.py \
  --model meta-llama/Meta-Llama-3-8B-Instruct \
  --backend vllm \
  --num-prompts 1000 \
  --input-len 512 \
  --output-len 128 \
  --tensor-parallel-size 1

# Sonraki çıktıya örnek:
# Throughput: 1423.2 tokens/s
# Request Throughput: 11.1 req/s

import asyncio
import time
import aiohttp
import statistics

async def send_request(session, url, prompt, req_id):
    start = time.perf_counter()
    first_token_time = None
    tokens = 0

    async with session.post(url, json={
        "model": "llama-3-8b",
        "messages": [{"role": "user", "content": prompt}],
        "max_tokens": 256,
        "stream": True,
    }) as resp:
        async for line in resp.content:
            line = line.decode().strip()
            if line.startswith("data: ") and line != "data: [DONE]":
                if first_token_time is None:
                    first_token_time = time.perf_counter()
                tokens += 1

    total_time = time.perf_counter() - start
    ttft = first_token_time - start if first_token_time else total_time
    return {
        "req_id": req_id,
        "ttft_ms": ttft * 1000,
        "total_ms": total_time * 1000,
        "tokens": tokens,
        "tps": tokens / total_time,
    }

async def benchmark(concurrency=10, total_requests=100):
    url = "http://localhost:8000/v1/chat/completions"
    prompt = "Makine öğrenmesinde overfitting nedir ve nasıl önlenir?"

    async with aiohttp.ClientSession() as session:
        semaphore = asyncio.Semaphore(concurrency)

        async def bounded_req(i):
            async with semaphore:
                return await send_request(session, url, prompt, i)

        start = time.perf_counter()
        results = await asyncio.gather(*[bounded_req(i) for i in range(total_requests)])
        elapsed = time.perf_counter() - start

    ttfts = [r["ttft_ms"] for r in results]
    tpss  = [r["tps"] for r in results]
    print(f"Toplam süre:     {elapsed:.1f}s")
    print(f"Request/s:       {total_requests/elapsed:.1f}")
    print(f"TTFT p50:        {statistics.median(ttfts):.0f} ms")
    print(f"TTFT p95:        {sorted(ttfts)[int(0.95*len(ttfts))]:.0f} ms")
    print(f"Throughput p50:  {statistics.median(tpss):.0f} tok/s")

asyncio.run(benchmark(concurrency=20, total_requests=200))

# 1. Model indir (HuggingFace Hub)
huggingface-cli login
huggingface-cli download meta-llama/Meta-Llama-3-8B-Instruct \
  --local-dir /models/llama-3-8b

# 2. vLLM sunucusu başlat
python -m vllm.entrypoints.openai.api_server \
  --model /models/llama-3-8b \
  --served-model-name llama-3-8b \
  --host 0.0.0.0 \
  --port 8000 \
  --gpu-memory-utilization 0.90 \
  --max-model-len 8192 \
  --enable-prefix-caching \
  --disable-log-requests \
  --tensor-parallel-size 1 &

# 3. Sağlık kontrolü
curl http://localhost:8000/health

# 4. Hızlı throughput testi
python benchmarks/benchmark_throughput.py \
  --model llama-3-8b \
  --backend openai \
  --endpoint http://localhost:8000/v1 \
  --num-prompts 200 \
  --request-rate 10

import asyncio
from fastapi import FastAPI
from fastapi.responses import StreamingResponse
from openai import AsyncOpenAI
from pydantic import BaseModel

app = FastAPI(title="LLM Proxy")
client = AsyncOpenAI(base_url="http://localhost:8000/v1", api_key="x")

class ChatRequest(BaseModel):
    message: str
    system: str = "Sen yardımcı bir asistansın."
    max_tokens: int = 512

@app.post("/chat/stream")
async def chat_stream(req: ChatRequest):
    async def event_generator():
        stream = await client.chat.completions.create(
            model="llama-3-8b",
            messages=[
                {"role": "system", "content": req.system},
                {"role": "user",   "content": req.message},
            ],
            max_tokens=req.max_tokens,
            stream=True,
        )
        async for chunk in stream:
            delta = chunk.choices[0].delta
            if delta.content:
                yield f"data: {delta.content}\n\n"
        yield "data: [DONE]\n\n"

    return StreamingResponse(event_generator(), media_type="text/event-stream")

# Çalıştır: uvicorn serve:app --host 0.0.0.0 --port 8080