Transformer
Dikkatin Anatomisi.

import torch
import torch.nn as nn

# ── RNN yaklaşımı: sequential, paralelleştirilemez ──────────
class SimpleRNN(nn.Module):
    def __init__(self, d_model):
        super().__init__()
        self.cell = nn.RNNCell(d_model, d_model)

    def forward(self, x):
        # x: (seq_len, batch, d_model)
        h = torch.zeros(x.size(1), x.size(2))
        outputs = []
        for t in range(x.size(0)):    # sekansiyel döngü
            h = self.cell(x[t], h)
            outputs.append(h)
        return torch.stack(outputs)

# ── Attention yaklaşımı: tüm token'lar aynı anda ───────────
def naive_attention(x):
    # x: (batch, seq_len, d_model)
    # Her token diğer tüm token'larla aynı anda ilişki kurar
    scores = torch.bmm(x, x.transpose(1, 2))          # (B, T, T)
    weights = torch.softmax(scores, dim=-1)
    return torch.bmm(weights, x)                       # (B, T, d)

# Test
batch, seq_len, d_model = 2, 10, 64
x = torch.randn(batch, seq_len, d_model)
out = naive_attention(x)
print(f"Girdi: {x.shape} → Çıktı: {out.shape}")
# Girdi: torch.Size([2, 10, 64]) → Çıktı: torch.Size([2, 10, 64])

import torch
import torch.nn as nn
import torch.nn.functional as F

class SelfAttention(nn.Module):
    """Manuel Q, K, V projeksiyon ile self-attention."""
    def __init__(self, d_model: int, d_k: int):
        super().__init__()
        self.d_k = d_k
        # Öğrenilen projeksiyon matrisleri
        self.W_q = nn.Linear(d_model, d_k, bias=False)
        self.W_k = nn.Linear(d_model, d_k, bias=False)
        self.W_v = nn.Linear(d_model, d_k, bias=False)

    def forward(self, x):
        # x: (batch, seq_len, d_model)
        Q = self.W_q(x)   # (B, T, d_k)  — sorgu vektörleri
        K = self.W_k(x)   # (B, T, d_k)  — anahtar vektörleri
        V = self.W_v(x)   # (B, T, d_k)  — değer vektörleri

        # Her Q ile tüm K'ların nokta çarpımı → benzerlik skoru
        scores = torch.bmm(Q, K.transpose(1, 2))          # (B, T, T)
        weights = F.softmax(scores, dim=-1)               # (B, T, T)
        out = torch.bmm(weights, V)                       # (B, T, d_k)
        return out, weights

# ─── Kullanım ───────────────────────────────────────────────
d_model, d_k = 128, 64
attn = SelfAttention(d_model, d_k)
x = torch.randn(2, 8, d_model)    # batch=2, seq=8
out, w = attn(x)
print(f"Çıktı şekli : {out.shape}")   # (2, 8, 64)
print(f"Ağırlık toplamı (satır başına): {w[0, 0].sum():.4f}")  # ≈ 1.0

import torch
import torch.nn.functional as F
import math

def scaled_dot_product_attention(Q, K, V, mask=None):
    """
    Q : (batch, heads, seq, d_k)
    K : (batch, heads, seq, d_k)
    V : (batch, heads, seq, d_v)
    mask: (batch, 1, seq, seq) — opsiyonel, -inf ile doldurulur
    """
    d_k = Q.size(-1)

    # Benzerlik skorları
    scores = torch.matmul(Q, K.transpose(-2, -1)) / math.sqrt(d_k)
    # scores: (batch, heads, seq_q, seq_k)

    # Mask uygulama (decoder causal mask veya padding mask)
    if mask is not None:
        scores = scores.masked_fill(mask == 0, float('-inf'))

    # Softmax: son eksende (seq_k boyutu üzerinde)
    attn_weights = F.softmax(scores, dim=-1)

    # Ağırlıklı değer toplamı
    output = torch.matmul(attn_weights, V)
    return output, attn_weights

# ─── Ölçeklemenin önemi — varyans karşılaştırması ───────────
d_k = 512
Q = torch.randn(1, 1, 10, d_k)
K = torch.randn(1, 1, 10, d_k)
V = torch.randn(1, 1, 10, d_k)

raw_scores    = torch.matmul(Q, K.transpose(-2, -1))
scaled_scores = raw_scores / math.sqrt(d_k)

print(f"Ham skor std    : {raw_scores.std():.2f}")      # ~22
print(f"Ölçekli skor std: {scaled_scores.std():.2f}")   # ~1

# Softmax entropisini karşılaştır
w_raw    = F.softmax(raw_scores[0,0,0], dim=-1)
w_scaled = F.softmax(scaled_scores[0,0,0], dim=-1)
entropy  = lambda p: -(p * (p + 1e-9).log()).sum()
print(f"Ham entropy     : {entropy(w_raw):.4f}")        # düşük → keskin
print(f"Ölçekli entropy : {entropy(w_scaled):.4f}")     # yüksek → dengeli

import torch
import torch.nn as nn
import torch.nn.functional as F
import math

class MultiHeadAttention(nn.Module):
    def __init__(self, d_model: int, num_heads: int):
        super().__init__()
        assert d_model % num_heads == 0, "d_model, num_heads'e tam bölünmeli"
        self.d_model    = d_model
        self.num_heads  = num_heads
        self.d_k        = d_model // num_heads   # her kafa boyutu

        # Tüm kafalar için tek bir büyük projeksiyon (verimli)
        self.W_q = nn.Linear(d_model, d_model, bias=False)
        self.W_k = nn.Linear(d_model, d_model, bias=False)
        self.W_v = nn.Linear(d_model, d_model, bias=False)
        self.W_o = nn.Linear(d_model, d_model, bias=False)

    def split_heads(self, x):
        # x: (B, T, d_model) → (B, h, T, d_k)
        B, T, _ = x.shape
        x = x.view(B, T, self.num_heads, self.d_k)
        return x.transpose(1, 2)

    def forward(self, x, mask=None):
        B, T, _ = x.shape
        Q = self.split_heads(self.W_q(x))   # (B, h, T, d_k)
        K = self.split_heads(self.W_k(x))
        V = self.split_heads(self.W_v(x))

        # Ölçekli nokta çarpımı attention
        scores = torch.matmul(Q, K.transpose(-2, -1)) / math.sqrt(self.d_k)
        if mask is not None:
            scores = scores.masked_fill(mask == 0, float('-inf'))
        weights = F.softmax(scores, dim=-1)
        context = torch.matmul(weights, V)               # (B, h, T, d_k)

        # Kafaları birleştir: (B, h, T, d_k) → (B, T, d_model)
        context = context.transpose(1, 2).contiguous()
        context = context.view(B, T, self.d_model)
        return self.W_o(context), weights

# ─── nn.MultiheadAttention ile karşılaştırma ────────────────
d_model, h = 256, 8
mha_custom = MultiHeadAttention(d_model, h)
mha_torch  = nn.MultiheadAttention(d_model, h, batch_first=True)

x = torch.randn(2, 16, d_model)
out_custom, _ = mha_custom(x)
out_torch,  _ = mha_torch(x, x, x)
print(f"Custom : {out_custom.shape}")   # (2, 16, 256)
print(f"PyTorch: {out_torch.shape}")    # (2, 16, 256)

import torch
import torch.nn as nn
import math

class SinusoidalPositionalEncoding(nn.Module):
    """Orijinal makaledeki sabit sinüsoidal PE."""
    def __init__(self, d_model: int, max_len: int = 5000, dropout: float = 0.1):
        super().__init__()
        self.dropout = nn.Dropout(p=dropout)

        # PE tablosunu önceden hesapla
        pe = torch.zeros(max_len, d_model)
        pos = torch.arange(0, max_len).unsqueeze(1)           # (max_len, 1)
        div = torch.exp(torch.arange(0, d_model, 2).float()
                        * (-math.log(10000.0) / d_model))    # (d_model/2,)
        pe[:, 0::2] = torch.sin(pos * div)                    # çift indisler
        pe[:, 1::2] = torch.cos(pos * div)                    # tek indisler
        pe = pe.unsqueeze(0)                                  # (1, max_len, d)
        self.register_buffer('pe', pe)                       # parametre değil

    def forward(self, x):
        # x: (batch, seq_len, d_model)
        x = x + self.pe[:, :x.size(1)]
        return self.dropout(x)

class LearnablePositionalEncoding(nn.Module):
    """BERT tarzı öğrenilebilir PE."""
    def __init__(self, d_model: int, max_len: int = 512):
        super().__init__()
        self.pe = nn.Embedding(max_len, d_model)

    def forward(self, x):
        B, T, _ = x.shape
        pos = torch.arange(T, device=x.device).unsqueeze(0)  # (1, T)
        return x + self.pe(pos)

# ─── PE değerlerini incele ───────────────────────────────────
pe_fixed = SinusoidalPositionalEncoding(d_model=64)
x = torch.zeros(1, 20, 64)
out = pe_fixed(x)     # PE değerleri gözlenebilir (x=0 olduğundan)
print(f"İlk token PE (ilk 4 değer): {out[0, 0, :4]}")
print(f"5. token PE  (ilk 4 değer): {out[0, 4, :4]}")

import torch
import torch.nn as nn

class FeedForward(nn.Module):
    """Position-wise FFN: genişle → aktivasyon → daralt."""
    def __init__(self, d_model: int, d_ff: int = None, dropout: float = 0.1):
        super().__init__()
        d_ff = d_ff or 4 * d_model       # varsayılan 4x genişleme
        self.net = nn.Sequential(
            nn.Linear(d_model, d_ff),
            nn.GELU(),                   # ReLU yerine GELU (modern tercih)
            nn.Dropout(dropout),
            nn.Linear(d_ff, d_model),
            nn.Dropout(dropout),
        )

    def forward(self, x):
        return self.net(x)

class FeedForwardSwiGLU(nn.Module):
    """LLaMA tarzı SwiGLU kapılı FFN.
    FFN(x) = (xW₁ ⊙ σ(xW_gate)) W₂
    """
    def __init__(self, d_model: int, d_ff: int = None):
        super().__init__()
        d_ff = d_ff or 4 * d_model
        self.gate  = nn.Linear(d_model, d_ff, bias=False)
        self.up    = nn.Linear(d_model, d_ff, bias=False)
        self.down  = nn.Linear(d_ff,    d_model, bias=False)
        self.act   = nn.SiLU()

    def forward(self, x):
        return self.down(self.act(self.gate(x)) * self.up(x))

# ─── Test ───────────────────────────────────────────────────
d_model = 512
ffn       = FeedForward(d_model)
ffn_swi   = FeedForwardSwiGLU(d_model)

x = torch.randn(2, 16, d_model)
print(f"Standart FFN  : {ffn(x).shape}")     # (2, 16, 512)
print(f"SwiGLU FFN    : {ffn_swi(x).shape}")  # (2, 16, 512)

# Parametre sayısı karşılaştırması
params = lambda m: sum(p.numel() for p in m.parameters())
print(f"Standart parametre: {params(ffn):,}")
print(f"SwiGLU   parametre: {params(ffn_swi):,}")

import torch
import torch.nn as nn

# ─── nn.LayerNorm kullanımı ──────────────────────────────────
d_model = 512
ln = nn.LayerNorm(d_model)    # normalize edilen boyut

x = torch.randn(2, 10, d_model)
out = ln(x)
print(f"Normalize edilmiş ortalama : {out[0, 0].mean():.6f}")   # ≈ 0
print(f"Normalize edilmiş std      : {out[0, 0].std():.6f}")    # ≈ 1

# ─── Manuel LayerNorm implementasyonu ───────────────────────
class ManualLayerNorm(nn.Module):
    def __init__(self, d_model: int, eps: float = 1e-5):
        super().__init__()
        self.eps   = eps
        self.gamma = nn.Parameter(torch.ones(d_model))   # scale
        self.beta  = nn.Parameter(torch.zeros(d_model))  # shift

    def forward(self, x):
        mean = x.mean(dim=-1, keepdim=True)
        var  = x.var(dim=-1, keepdim=True, unbiased=False)
        x_hat = (x - mean) / (var + self.eps).sqrt()
        return self.gamma * x_hat + self.beta

# ─── Pre-LN vs Post-LN karşılaştırması ─────────────────────
class PostLNBlock(nn.Module):
    def __init__(self, d_model, sublayer):
        super().__init__()
        self.sublayer = sublayer
        self.norm     = nn.LayerNorm(d_model)

    def forward(self, x):
        return self.norm(x + self.sublayer(x))   # Post-LN: norm sonra

class PreLNBlock(nn.Module):
    def __init__(self, d_model, sublayer):
        super().__init__()
        self.sublayer = sublayer
        self.norm     = nn.LayerNorm(d_model)

    def forward(self, x):
        return x + self.sublayer(self.norm(x))   # Pre-LN: norm önce

import torch
import torch.nn as nn

class TransformerBlock(nn.Module):
    """Pre-LN transformer bloğu: Attention + FFN + residuals."""
    def __init__(self, d_model: int, num_heads: int, d_ff: int, dropout: float = 0.1):
        super().__init__()
        self.norm1 = nn.LayerNorm(d_model)
        self.norm2 = nn.LayerNorm(d_model)
        self.attn  = nn.MultiheadAttention(d_model, num_heads,
                                              dropout=dropout, batch_first=True)
        self.ffn   = nn.Sequential(
            nn.Linear(d_model, d_ff),
            nn.GELU(),
            nn.Dropout(dropout),
            nn.Linear(d_ff, d_model),
            nn.Dropout(dropout),
        )

    def forward(self, x, mask=None):
        # Pre-LN + Residual — Attention alt katmanı
        normed = self.norm1(x)
        attn_out, _ = self.attn(normed, normed, normed,
                                  attn_mask=mask,
                                  need_weights=False)
        x = x + attn_out                          # residual #1

        # Pre-LN + Residual — FFN alt katmanı
        x = x + self.ffn(self.norm2(x))           # residual #2
        return x

# ─── Gradient akışını doğrula ───────────────────────────────
block = TransformerBlock(d_model=128, num_heads=4, d_ff=512)
x = torch.randn(1, 8, 128, requires_grad=True)
out = block(x)
loss = out.sum()
loss.backward()
print(f"Giriş gradyanı norm: {x.grad.norm():.4f}")   # sıfır değil

import torch
import torch.nn as nn

class TransformerEncoder(nn.Module):
    """BERT tarzı bidirectional transformer encoder."""
    def __init__(
        self,
        vocab_size: int,
        d_model:    int = 512,
        num_heads:  int = 8,
        num_layers: int = 6,
        d_ff:       int = 2048,
        max_len:    int = 512,
        dropout:    float = 0.1,
    ):
        super().__init__()
        self.token_emb = nn.Embedding(vocab_size, d_model)
        self.pos_emb   = nn.Embedding(max_len, d_model)
        self.drop      = nn.Dropout(dropout)
        self.layers    = nn.ModuleList([
            _EncoderLayer(d_model, num_heads, d_ff, dropout)
            for _ in range(num_layers)
        ])
        self.norm = nn.LayerNorm(d_model)

    def forward(self, token_ids, padding_mask=None):
        # token_ids: (batch, seq_len) — tam sayı indeksler
        B, T = token_ids.shape
        pos  = torch.arange(T, device=token_ids.device).unsqueeze(0)
        x    = self.drop(self.token_emb(token_ids) + self.pos_emb(pos))

        for layer in self.layers:
            x = layer(x, padding_mask)

        x = self.norm(x)
        cls_repr = x[:, 0]    # [CLS] token temsili — sınıflandırma için
        return x, cls_repr

class _EncoderLayer(nn.Module):
    def __init__(self, d_model, num_heads, d_ff, dropout):
        super().__init__()
        self.norm1 = nn.LayerNorm(d_model)
        self.norm2 = nn.LayerNorm(d_model)
        self.attn  = nn.MultiheadAttention(d_model, num_heads,
                                              dropout=dropout, batch_first=True)
        self.ffn   = nn.Sequential(
            nn.Linear(d_model, d_ff), nn.GELU(), nn.Dropout(dropout),
            nn.Linear(d_ff, d_model), nn.Dropout(dropout),
        )

    def forward(self, x, mask=None):
        a, _ = self.attn(self.norm1(x), self.norm1(x), self.norm1(x),
                          key_padding_mask=mask, need_weights=False)
        x = x + a
        x = x + self.ffn(self.norm2(x))
        return x

# ─── BERT benzeri sınıflandırma başlığı ─────────────────────
class BERTClassifier(nn.Module):
    def __init__(self, encoder, num_classes):
        super().__init__()
        self.encoder = encoder
        self.head = nn.Linear(encoder.token_emb.embedding_dim, num_classes)

    def forward(self, token_ids):
        _, cls = self.encoder(token_ids)
        return self.head(cls)    # (batch, num_classes)

# Test
encoder = TransformerEncoder(vocab_size=30000, d_model=256, num_layers=4)
ids = torch.randint(0, 30000, (2, 32))
seq_out, cls_out = encoder(ids)
print(f"Dizi temsili : {seq_out.shape}")   # (2, 32, 256)
print(f"CLS temsili  : {cls_out.shape}")   # (2, 256)

import torch
import torch.nn as nn
import torch.nn.functional as F

def causal_mask(seq_len: int, device='cpu'):
    """Alt üçgen mask: True → attend, False → blokla."""
    mask = torch.tril(torch.ones(seq_len, seq_len, device=device))
    return mask.bool()

class CausalSelfAttention(nn.Module):
    def __init__(self, d_model: int, num_heads: int, dropout: float = 0.1):
        super().__init__()
        self.d_model   = d_model
        self.num_heads = num_heads
        self.d_k       = d_model // num_heads
        self.qkv_proj  = nn.Linear(d_model, 3 * d_model, bias=False)
        self.out_proj  = nn.Linear(d_model, d_model, bias=False)
        self.drop      = nn.Dropout(dropout)

    def forward(self, x):
        B, T, C = x.shape
        # Q, K, V tek bir projeksiyondan split
        qkv = self.qkv_proj(x).split(self.d_model, dim=2)
        Q, K, V = [t.view(B, T, self.num_heads, self.d_k).transpose(1, 2)
                   for t in qkv]          # her biri (B, h, T, d_k)

        # PyTorch 2.0+ Flash Attention — causal=True ile causal mask otomatik
        out = F.scaled_dot_product_attention(
            Q, K, V,
            dropout_p=self.drop.p if self.training else 0.0,
            is_causal=True,
        )
        out = out.transpose(1, 2).contiguous().view(B, T, C)
        return self.out_proj(out)

class GPTDecoderLayer(nn.Module):
    def __init__(self, d_model, num_heads, d_ff, dropout=0.1):
        super().__init__()
        self.norm1 = nn.LayerNorm(d_model)
        self.norm2 = nn.LayerNorm(d_model)
        self.attn  = CausalSelfAttention(d_model, num_heads, dropout)
        self.ffn   = nn.Sequential(
            nn.Linear(d_model, d_ff), nn.GELU(), nn.Dropout(dropout),
            nn.Linear(d_ff, d_model), nn.Dropout(dropout),
        )

    def forward(self, x):
        x = x + self.attn(self.norm1(x))
        x = x + self.ffn(self.norm2(x))
        return x

# ─── Causal mask değerlerini kontrol et ─────────────────────
mask = causal_mask(5)
print(mask.int())
# tensor([[1, 0, 0, 0, 0],
#         [1, 1, 0, 0, 0],
#         [1, 1, 1, 0, 0],
#         [1, 1, 1, 1, 0],
#         [1, 1, 1, 1, 1]])

import torch
import torch.nn as nn

class CrossAttention(nn.Module):
    """Decoder'ın encoder çıktısını sorguladığı cross-attention."""
    def __init__(self, d_model: int, num_heads: int, dropout: float = 0.1):
        super().__init__()
        self.attn = nn.MultiheadAttention(d_model, num_heads,
                                            dropout=dropout, batch_first=True)

    def forward(self, decoder_x, encoder_out):
        # decoder_x  : (B, T_tgt, d_model)  — Query kaynağı
        # encoder_out: (B, T_src, d_model)  — Key & Value kaynağı
        out, attn_weights = self.attn(
            query = decoder_x,
            key   = encoder_out,
            value = encoder_out,
        )
        return out, attn_weights

class Seq2SeqDecoderLayer(nn.Module):
    """Causal self-attention + cross-attention + FFN."""
    def __init__(self, d_model, num_heads, d_ff, dropout=0.1):
        super().__init__()
        self.norm1 = nn.LayerNorm(d_model)
        self.norm2 = nn.LayerNorm(d_model)
        self.norm3 = nn.LayerNorm(d_model)
        self.self_attn  = nn.MultiheadAttention(d_model, num_heads,
                                                   dropout=dropout, batch_first=True)
        self.cross_attn = CrossAttention(d_model, num_heads, dropout)
        self.ffn = nn.Sequential(
            nn.Linear(d_model, d_ff), nn.GELU(), nn.Dropout(dropout),
            nn.Linear(d_ff, d_model), nn.Dropout(dropout),
        )

    def forward(self, x, encoder_out, causal_mask=None):
        # 1. Causal self-attention (yalnızca geçmiş decoder token'ları)
        normed = self.norm1(x)
        sa_out, _ = self.self_attn(normed, normed, normed,
                                     attn_mask=causal_mask, need_weights=False)
        x = x + sa_out

        # 2. Cross-attention (encoder çıktısını sorgula)
        ca_out, _ = self.cross_attn(self.norm2(x), encoder_out)
        x = x + ca_out

        # 3. FFN
        x = x + self.ffn(self.norm3(x))
        return x

# ─── Seq2Seq forward pass örneği ────────────────────────────
d_model, h = 256, 8
src_len, tgt_len = 20, 15
B = 2

encoder_out = torch.randn(B, src_len, d_model)
decoder_in  = torch.randn(B, tgt_len, d_model)

T = tgt_len
causal = torch.triu(torch.ones(T, T) * float('-inf'), diagonal=1)

dec_layer = Seq2SeqDecoderLayer(d_model, h, d_ff=1024)
out = dec_layer(decoder_in, encoder_out, causal_mask=causal)
print(f"Decoder çıktısı: {out.shape}")   # (2, 15, 256)

import torch
import torch.nn as nn
import torch.nn.functional as F

class MiniGPTConfig:
    vocab_size  = 65        # Shakespeare karakter seti boyutu
    block_size  = 128       # max bağlam uzunluğu (context window)
    d_model     = 192       # embedding boyutu
    num_heads   = 6
    num_layers  = 4
    d_ff        = 768        # 4 * d_model
    dropout     = 0.1

class CausalBlock(nn.Module):
    def __init__(self, cfg):
        super().__init__()
        self.norm1 = nn.LayerNorm(cfg.d_model)
        self.norm2 = nn.LayerNorm(cfg.d_model)
        self.attn  = nn.MultiheadAttention(
            cfg.d_model, cfg.num_heads,
            dropout=cfg.dropout, batch_first=True
        )
        self.ffn = nn.Sequential(
            nn.Linear(cfg.d_model, cfg.d_ff), nn.GELU(), nn.Dropout(cfg.dropout),
            nn.Linear(cfg.d_ff, cfg.d_model), nn.Dropout(cfg.dropout),
        )

    def forward(self, x, causal_mask):
        B, T, _ = x.shape
        n = self.norm1(x)
        a, _ = self.attn(n, n, n, attn_mask=causal_mask, need_weights=False)
        x = x + a
        x = x + self.ffn(self.norm2(x))
        return x

class MiniGPT(nn.Module):
    def __init__(self, cfg):
        super().__init__()
        self.cfg = cfg
        self.tok_emb = nn.Embedding(cfg.vocab_size, cfg.d_model)
        self.pos_emb = nn.Embedding(cfg.block_size, cfg.d_model)
        self.drop    = nn.Dropout(cfg.dropout)
        self.blocks  = nn.ModuleList([CausalBlock(cfg) for _ in range(cfg.num_layers)])
        self.norm    = nn.LayerNorm(cfg.d_model)
        self.head    = nn.Linear(cfg.d_model, cfg.vocab_size, bias=False)

        # Token embedding ve output projeksiyon ağırlıklarını paylaş (weight tying)
        self.head.weight = self.tok_emb.weight

        self.apply(self._init_weights)

    def _init_weights(self, module):
        if isinstance(module, (nn.Linear, nn.Embedding)):
            nn.init.normal_(module.weight, mean=0.0, std=0.02)
        if isinstance(module, nn.Linear) and module.bias is not None:
            nn.init.zeros_(module.bias)

    def forward(self, idx, targets=None):
        # idx: (B, T) — token indeksleri
        B, T = idx.shape
        pos  = torch.arange(T, device=idx.device)

        x = self.drop(self.tok_emb(idx) + self.pos_emb(pos))

        # Causal mask: üst üçgen -inf
        mask = torch.triu(torch.ones(T, T, device=idx.device) * float('-inf'),
                           diagonal=1)
        for block in self.blocks:
            x = block(x, mask)

        x      = self.norm(x)
        logits = self.head(x)   # (B, T, vocab_size)

        loss = None
        if targets is not None:
            # Cross entropy: logits → (B*T, V), targets → (B*T,)
            loss = F.cross_entropy(logits.view(-1, self.cfg.vocab_size),
                                     targets.view(-1))
        return logits, loss

    @torch.no_grad()
    def generate(self, idx, max_new_tokens: int, temperature: float = 1.0):
        for _ in range(max_new_tokens):
            # Bağlam penceresini aş
            idx_cond = idx[:, -self.cfg.block_size:]
            logits, _ = self(idx_cond)
            logits = logits[:, -1] / temperature       # son token logit'i
            probs  = F.softmax(logits, dim=-1)
            nxt    = torch.multinomial(probs, num_samples=1)
            idx    = torch.cat([idx, nxt], dim=1)
        return idx

# ─── Model istatistikleri ────────────────────────────────────
cfg   = MiniGPTConfig()
model = MiniGPT(cfg)
total = sum(p.numel() for p in model.parameters())
print(f"Toplam parametre: {total:,}")      # ~1.5M

# Forward pass
idx     = torch.randint(0, cfg.vocab_size, (4, 32))
targets = torch.randint(0, cfg.vocab_size, (4, 32))
logits, loss = model(idx, targets)
print(f"Logits: {logits.shape}, Loss: {loss.item():.4f}")

# Üretim örneği
prompt = torch.zeros(1, 1, dtype=torch.long)
gen    = model.generate(prompt, max_new_tokens=20)
print(f"Üretilen token ID'leri: {gen[0].tolist()}")