RL Temelleri
MDP & Q-Learning.

import gymnasium as gym
import numpy as np

# FrozenLake: 4x4 grid, 16 state, 4 action (L/D/R/U)
env = gym.make('FrozenLake-v1', is_slippery=False)

print(f"State space  : {env.observation_space}")   # Discrete(16)
print(f"Action space : {env.action_space}")         # Discrete(4)

# MDP bileşenlerini incele
# P[state][action] = [(prob, next_state, reward, done), ...]
transition = env.unwrapped.P[0][2]   # state=0, action=RIGHT
print(f"Geçiş: {transition}")
# [(1.0, 1, 0.0, False)]  — deterministik

# Episode başlatma ve adım atma
state, info = env.reset(seed=42)
print(f"Başlangıç state: {state}")      # 0 (sol üst köşe)

total_reward = 0
for _ in range(10):
    action = env.action_space.sample()   # rastgele eylem
    next_state, reward, terminated, truncated, info = env.step(action)
    total_reward += reward
    print(f"  s={state} a={action} → s'={next_state} r={reward} done={terminated}")
    state = next_state
    if terminated or truncated:
        break

env.close()

import gymnasium as gym
import numpy as np

def value_iteration(env, gamma=0.99, theta=1e-8):
    """
    Bellman optimallik denklemiyle V*(s) hesapla.
    Transition tablosuna erişim gerektirir (model-based).
    """
    n_states  = env.observation_space.n
    n_actions = env.action_space.n
    V = np.zeros(n_states)

    iteration = 0
    while True:
        delta = 0
        for s in range(n_states):
            v_old = V[s]
            action_values = []
            for a in range(n_actions):
                q = 0
                for prob, s_next, reward, done in env.unwrapped.P[s][a]:
                    q += prob * (reward + gamma * V[s_next] * (1 - done))
                action_values.append(q)
            V[s] = max(action_values)
            delta = max(delta, abs(v_old - V[s]))
        iteration += 1
        if delta < theta:
            break

    # Optimal policy çıkar
    policy = np.zeros(n_states, dtype=int)
    for s in range(n_states):
        q_vals = []
        for a in range(n_actions):
            q = sum(prob * (r + gamma * V[s_] * (1-d))
                    for prob, s_, r, d in env.unwrapped.P[s][a])
            q_vals.append(q)
        policy[s] = np.argmax(q_vals)

    print(f"Value iteration {iteration} iterasyonda converge etti.")
    return V, policy

env = gym.make('FrozenLake-v1', is_slippery=False)
V_star, pi_star = value_iteration(env)
print("V*(s) =\n", V_star.reshape(4,4).round(3))
print("π*(s) =\n", pi_star.reshape(4,4))
# 0=L 1=D 2=R 3=U

import gymnasium as gym
import numpy as np

def q_learning(env_name='FrozenLake-v1',
               n_episodes=5000,
               alpha=0.1,
               gamma=0.99,
               eps_start=1.0,
               eps_end=0.01,
               eps_decay=0.995):

    env = gym.make(env_name, is_slippery=False)
    n_s = env.observation_space.n
    n_a = env.action_space.n
    Q   = np.zeros((n_s, n_a))     # Q-tablo başlangıcı

    eps      = eps_start
    rewards  = []

    for ep in range(n_episodes):
        state, _ = env.reset()
        total_r   = 0

        while True:
            # Epsilon-greedy eylem seçimi
            if np.random.random() < eps:
                action = env.action_space.sample()   # keşif
            else:
                action = np.argmax(Q[state])          # sömürü

            next_state, reward, terminated, truncated, _ = env.step(action)

            # Q-learning güncellemesi
            td_target = reward + gamma * np.max(Q[next_state]) * (1 - terminated)
            td_error  = td_target - Q[state, action]
            Q[state, action] += alpha * td_error

            total_r += reward
            state    = next_state

            if terminated or truncated:
                break

        eps = max(eps_end, eps * eps_decay)
        rewards.append(total_r)

        if (ep + 1) % 500 == 0:
            win_rate = np.mean(rewards[-500:])
            print(f"Ep {ep+1:5d} | Win rate: {win_rate:.3f} | eps: {eps:.3f}")

    env.close()
    return Q, rewards

Q, rewards = q_learning()
print("\nÖğrenilen Q-tablo:")
print(Q.round(2))

import numpy as np
import matplotlib.pyplot as plt

def epsilon_schedule(n_steps=10_000, eps_start=1.0, eps_end=0.05):
    """Farklı decay stratejilerini karşılaştır."""
    steps = np.arange(n_steps)

    # 1. Exponential decay
    decay = (eps_end / eps_start) ** (1 / n_steps)
    eps_exp = eps_start * (decay ** steps)
    eps_exp = np.maximum(eps_exp, eps_end)

    # 2. Linear decay
    eps_lin = np.linspace(eps_start, eps_end, n_steps)

    # 3. Cosine annealing
    eps_cos = eps_end + 0.5 * (eps_start - eps_end) * (
        1 + np.cos(np.pi * steps / n_steps))

    plt.figure(figsize=(10, 4))
    plt.plot(steps, eps_exp, label='Exponential', linewidth=2)
    plt.plot(steps, eps_lin, label='Linear',      linewidth=2)
    plt.plot(steps, eps_cos, label='Cosine',      linewidth=2)
    plt.xlabel('Adım'); plt.ylabel('ε')
    plt.title('Epsilon Decay Stratejileri')
    plt.legend(); plt.tight_layout(); plt.show()

epsilon_schedule()

import gymnasium as gym
import numpy as np

def td0_prediction(env, policy, n_episodes=2000, alpha=0.05, gamma=0.99):
    """TD(0) ile policy evaluation — V^π tahmini."""
    n_s = env.observation_space.n
    V   = np.zeros(n_s)

    for _ in range(n_episodes):
        s, _ = env.reset()
        while True:
            a = policy[s]
            s_next, r, done, trunc, _ = env.step(a)
            # TD(0) güncellemesi
            V[s] += alpha * (r + gamma * V[s_next] * (1 - done) - V[s])
            s = s_next
            if done or trunc:
                break
    return V

def mc_prediction(env, policy, n_episodes=2000, gamma=0.99):
    """Monte Carlo ile policy evaluation — tam episode gerektirir."""
    n_s = env.observation_space.n
    V       = np.zeros(n_s)
    returns = [[] for _ in range(n_s)]

    for _ in range(n_episodes):
        traj = []
        s, _ = env.reset()
        while True:
            a = policy[s]
            s_next, r, done, trunc, _ = env.step(a)
            traj.append((s, r))
            s = s_next
            if done or trunc:
                break
        # Geriye doğru return hesapla
        G = 0
        for s_t, r_t in reversed(traj):
            G = r_t + gamma * G
            returns[s_t].append(G)
            V[s_t] = np.mean(returns[s_t])
    return V

import gymnasium as gym
import numpy as np

def td_lambda(env, policy, lam=0.9, alpha=0.05, gamma=0.99, n_episodes=2000):
    """TD(λ) accumulating traces ile V^π tahmini."""
    n_s = env.observation_space.n
    V = np.zeros(n_s)

    for _ in range(n_episodes):
        e = np.zeros(n_s)     # eligibility traces sıfırla
        s, _ = env.reset()

        while True:
            a = policy[s]
            s_next, r, done, trunc, _ = env.step(a)

            # TD error
            delta = r + gamma * V[s_next] * (1 - done) - V[s]

            # Trace güncelleme
            e *= gamma * lam
            e[s] += 1                  # accumulating trace

            # Tüm state'leri güncelle
            V += alpha * delta * e

            s = s_next
            if done or trunc:
                break

    return V

env = gym.make('FrozenLake-v1', is_slippery=False)
# Basit deterministik policy: sağ-aşağı stratejisi
policy = np.array([2,2,1,0, 1,0,1,0, 2,1,1,0, 0,2,2,0])
V = td_lambda(env, policy, lam=0.9)
print("V^π (TD-λ=0.9):\n", V.reshape(4,4).round(3))

import gymnasium as gym
import numpy as np
import matplotlib.pyplot as plt

# State ayrıklaştırma parametreleri
N_BINS    = [6, 12, 6, 12]    # her boyut için bin sayısı
STATE_LOW  = np.array([-2.4, -3.0, -0.21, -3.0])
STATE_HIGH = np.array([ 2.4,  3.0,  0.21,  3.0])

BINS = [np.linspace(STATE_LOW[i], STATE_HIGH[i], N_BINS[i] - 1)
        for i in range(4)]

def discretize(obs):
    idx = tuple(np.digitize(obs[i], BINS[i]) for i in range(4))
    return idx

# Q-tablo: N_BINS x 2 actions
Q = np.zeros(N_BINS + [2])

env       = gym.make('CartPole-v1')
alpha     = 0.1
gamma     = 0.99
eps       = 1.0
eps_decay = 0.995
rewards   = []

for ep in range(3000):
    obs, _ = env.reset()
    state  = discretize(obs)
    total  = 0

    while True:
        if np.random.random() < eps:
            action = env.action_space.sample()
        else:
            action = np.argmax(Q[state])

        obs2, r, done, trunc, _ = env.step(action)
        state2 = discretize(obs2)

        best_next = np.max(Q[state2])
        Q[state + (action,)] += alpha * (r + gamma * best_next * (1-done) - Q[state + (action,)])

        state = state2
        total += r
        if done or trunc:
            break

    eps = max(0.01, eps * eps_decay)
    rewards.append(total)

# Smoothed learning curve
window = 100
avg = np.convolve(rewards, np.ones(window)/window, mode='valid')
plt.plot(avg); plt.xlabel('Episode')
plt.ylabel('Ortalama Ödül (100-ep)')
plt.title('CartPole Q-Tablo Learning Curve')
plt.axhline(475, color='r', linestyle='--', label='Çözüm eşiği')
plt.legend(); plt.tight_layout(); plt.show()
print(f"Son 100 ep ort: {np.mean(rewards[-100:]):.1f}")

import gymnasium as gym
import numpy as np
import matplotlib.pyplot as plt

def solve_frozenlake(
        is_slippery: bool = True,
        n_episodes: int  = 20_000,
        alpha:      float = 0.1,
        gamma:      float = 0.99,
        eps_start:  float = 1.0,
        eps_end:    float = 0.01,
        eps_decay:  float = 0.9995):

    env = gym.make('FrozenLake-v1', is_slippery=is_slippery)
    Q   = np.zeros((env.observation_space.n, env.action_space.n))
    eps = eps_start
    win_history, eps_history = [], []

    for ep in range(n_episodes):
        s, _ = env.reset()
        won = False

        while True:
            a = (env.action_space.sample() if np.random.random() < eps
                 else np.argmax(Q[s]))
            s2, r, done, trunc, _ = env.step(a)
            Q[s, a] += alpha * (r + gamma * np.max(Q[s2]) * (1-done) - Q[s, a])
            if done and r == 1:
                won = True
            s = s2
            if done or trunc:
                break

        eps = max(eps_end, eps * eps_decay)
        win_history.append(int(won))
        eps_history.append(eps)

    # ── Görselleştirme ──────────────────────────────────────
    window = 500
    win_rate = np.convolve(win_history,
                            np.ones(window)/window, mode='valid')

    fig, axes = plt.subplots(1, 3, figsize=(15, 4))

    # 1. Win rate curve
    axes[0].plot(win_rate, color='steelblue')
    axes[0].axhline(0.78, color='orange', linestyle='--', label='Çözüm eşiği')
    axes[0].set_title('Win Rate'); axes[0].legend()

    # 2. Epsilon curve
    axes[1].plot(eps_history, color='coral')
    axes[1].set_title('Epsilon Decay')

    # 3. Optimal policy heatmap
    policy = np.argmax(Q, axis=1).reshape(4, 4)
    arrows = {0:'←', 1:'↓', 2:'→', 3:'↑'}
    im = axes[2].imshow(np.max(Q, axis=1).reshape(4,4), cmap='YlGnBu')
    for r in range(4):
        for c in range(4):
            axes[2].text(c, r, arrows[policy[r, c]],
                         ha='center', va='center', fontsize=18)
    axes[2].set_title('Optimal Policy')
    fig.colorbar(im, ax=axes[2])

    plt.tight_layout(); plt.show()
    print(f"Son 1000 ep win rate: {np.mean(win_history[-1000:]):.3f}")
    return Q

# Slippery (stokastik) ortam daha zor — daha fazla episode gerekir
Q = solve_frozenlake(is_slippery=True, n_episodes=20_000)