Deep RL
DQN, A2C, PPO.

import torch
import torch.nn as nn
import torch.optim as optim
import numpy as np
from collections import deque
import random

class QNetwork(nn.Module):
    def __init__(self, obs_dim, n_actions, hidden=128):
        super().__init__()
        self.net = nn.Sequential(
            nn.Linear(obs_dim, hidden), nn.ReLU(),
            nn.Linear(hidden, hidden),  nn.ReLU(),
            nn.Linear(hidden, n_actions)
        )
    def forward(self, x):
        return self.net(x)

class ReplayBuffer:
    def __init__(self, capacity=50_000):
        self.buf = deque(maxlen=capacity)
    def push(self, *args):
        self.buf.append(args)
    def sample(self, batch_size):
        batch = random.sample(self.buf, batch_size)
        s,a,r,s2,d = zip(*batch)
        return (torch.FloatTensor(np.array(s)),
                torch.LongTensor(a),
                torch.FloatTensor(r),
                torch.FloatTensor(np.array(s2)),
                torch.FloatTensor(d))
    def __len__(self): return len(self.buf)

class DQNAgent:
    def __init__(self, obs_dim, n_actions, lr=1e-3, gamma=0.99,
                 batch_size=64, target_update=500):
        self.n_actions    = n_actions
        self.gamma        = gamma
        self.batch_size   = batch_size
        self.target_update = target_update
        self.step_count   = 0

        self.online = QNetwork(obs_dim, n_actions)
        self.target = QNetwork(obs_dim, n_actions)
        self.target.load_state_dict(self.online.state_dict())
        self.target.eval()

        self.opt    = optim.Adam(self.online.parameters(), lr=lr)
        self.buffer = ReplayBuffer()

    def act(self, state, eps):
        if np.random.random() < eps:
            return np.random.randint(self.n_actions)
        with torch.no_grad():
            q = self.online(torch.FloatTensor(state).unsqueeze(0))
        return q.argmax().item()

    def update(self):
        if len(self.buffer) < self.batch_size:
            return None
        s, a, r, s2, d = self.buffer.sample(self.batch_size)

        # Current Q values
        q_vals = self.online(s).gather(1, a.unsqueeze(1)).squeeze()

        # Target Q values (target network, no grad)
        with torch.no_grad():
            next_q = self.target(s2).max(1)[0]
            td_target = r + self.gamma * next_q * (1 - d)

        loss = nn.functional.mse_loss(q_vals, td_target)
        self.opt.zero_grad(); loss.backward(); self.opt.step()

        self.step_count += 1
        if self.step_count % self.target_update == 0:
            self.target.load_state_dict(self.online.state_dict())

        return loss.item()

import torch
import torch.nn as nn

class DuelingQNetwork(nn.Module):
    """Dueling DQN: Q(s,a) = V(s) + A(s,a) - mean(A(s,·))"""
    def __init__(self, obs_dim, n_actions, hidden=256):
        super().__init__()
        self.feature = nn.Sequential(
            nn.Linear(obs_dim, hidden), nn.ReLU()
        )
        # Value stream: V(s)
        self.value_stream = nn.Sequential(
            nn.Linear(hidden, hidden), nn.ReLU(),
            nn.Linear(hidden, 1)
        )
        # Advantage stream: A(s,a)
        self.adv_stream = nn.Sequential(
            nn.Linear(hidden, hidden), nn.ReLU(),
            nn.Linear(hidden, n_actions)
        )

    def forward(self, x):
        feat = self.feature(x)
        V    = self.value_stream(feat)          # (B, 1)
        A    = self.adv_stream(feat)            # (B, n_actions)
        # Q(s,a) = V(s) + A(s,a) - mean_a A(s,a)
        Q = V + A - A.mean(dim=1, keepdim=True)
        return Q

# Double DQN target hesaplama
def double_dqn_target(online_net, target_net, s2, r, d, gamma=0.99):
    with torch.no_grad():
        # Aksiyon seçimi: online network
        best_actions = online_net(s2).argmax(dim=1, keepdim=True)
        # Değerlendirme: target network
        next_q = target_net(s2).gather(1, best_actions).squeeze()
        target = r + gamma * next_q * (1 - d)
    return target

import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.distributions import Categorical

class ActorCritic(nn.Module):
    """Paylaşımlı feature extraction, ayrı actor/critic head."""
    def __init__(self, obs_dim, n_actions, hidden=256):
        super().__init__()
        self.shared = nn.Sequential(
            nn.Linear(obs_dim, hidden), nn.Tanh(),
            nn.Linear(hidden, hidden),  nn.Tanh()
        )
        self.actor  = nn.Linear(hidden, n_actions)   # logits
        self.critic = nn.Linear(hidden, 1)           # V(s)

    def forward(self, x):
        feat   = self.shared(x)
        logits = self.actor(feat)
        value  = self.critic(feat).squeeze(-1)
        dist   = Categorical(logits=logits)
        return dist, value

    def get_action(self, state):
        state = torch.FloatTensor(state).unsqueeze(0)
        dist, value = self.forward(state)
        action = dist.sample()
        log_prob = dist.log_prob(action)
        return action.item(), log_prob, value

import torch
import torch.nn.functional as F

def a2c_loss(log_probs, values, rewards, dones,
             gamma=0.99, entropy_coef=0.01, value_coef=0.5):
    """
    A2C güncelleme adımı.
    log_probs : (T,)  alınan aksiyonların log olasılıkları
    values    : (T,)  critic tahminleri V(s_t)
    rewards   : (T,)  ortamdan alınan ödüller
    dones     : (T,)  episode bitiş bayrakları
    """
    T = len(rewards)

    # Discounted returns hesapla (son state'ten geriye)
    returns = []
    G = 0
    for r, d in zip(reversed(rewards), reversed(dones)):
        G = r + gamma * G * (1 - d)
        returns.insert(0, G)
    returns = torch.FloatTensor(returns)

    # Advantage = return - baseline (normalize edilmiş)
    advantages = returns - values.detach()
    advantages = (advantages - advantages.mean()) / (advantages.std() + 1e-8)

    # Policy loss (actor)
    policy_loss = -(log_probs * advantages).mean()

    # Value loss (critic)
    value_loss = F.mse_loss(values, returns)

    # Entropy bonus (keşif teşviki)
    # (entropy ayrıca gerekirse distribution'dan hesaplanır)

    total_loss = policy_loss + value_coef * value_loss
    return total_loss, policy_loss.item(), value_loss.item()

import torch
import torch.nn.functional as F

def compute_gae(rewards, values, dones, gamma=0.99, lam=0.95):
    """
    GAE (Generalized Advantage Estimation) hesapla.
    values: (T+1,) — son state için bootstrap değeri dahil.
    """
    advantages = []
    gae = 0
    for t in reversed(range(len(rewards))):
        delta  = rewards[t] + gamma * values[t+1] * (1-dones[t]) - values[t]
        gae    = delta + gamma * lam * (1-dones[t]) * gae
        advantages.insert(0, gae)
    advantages = torch.FloatTensor(advantages)
    returns    = advantages + torch.FloatTensor(values[:-1])
    return advantages, returns

def ppo_update(actor_critic, optimizer, states, actions, old_log_probs,
               advantages, returns, clip_eps=0.2, value_coef=0.5,
               entropy_coef=0.01, n_epochs=10, minibatch_size=64):
    """PPO güncelleme — n_epochs kez minibatch üzerinde."""
    dataset_size = states.shape[0]
    advantages = (advantages - advantages.mean()) / (advantages.std() + 1e-8)

    total_loss_log = []
    for _ in range(n_epochs):
        idx = torch.randperm(dataset_size)
        for start in range(0, dataset_size, minibatch_size):
            mb_idx = idx[start:start+minibatch_size]
            mb_s   = states[mb_idx]
            mb_a   = actions[mb_idx]
            mb_adv = advantages[mb_idx]
            mb_ret = returns[mb_idx]
            mb_lp  = old_log_probs[mb_idx]

            dist, values = actor_critic(mb_s)
            new_log_probs = dist.log_prob(mb_a)
            entropy       = dist.entropy().mean()

            # Clipped surrogate loss
            ratio        = (new_log_probs - mb_lp).exp()
            surr1        = ratio * mb_adv
            surr2        = torch.clamp(ratio, 1-clip_eps, 1+clip_eps) * mb_adv
            policy_loss  = -torch.min(surr1, surr2).mean()

            value_loss   = F.mse_loss(values, mb_ret)

            loss = policy_loss + value_coef * value_loss - entropy_coef * entropy
            optimizer.zero_grad(); loss.backward(); optimizer.step()
            total_loss_log.append(loss.item())

    return sum(total_loss_log) / len(total_loss_log)

from stable_baselines3 import PPO
from stable_baselines3.common.env_util import make_vec_env
from stable_baselines3.common.callbacks import EvalCallback
from stable_baselines3.common.monitor import Monitor
import gymnasium as gym

# Vektörel ortam: 8 paralel env
vec_env = make_vec_env("CartPole-v1", n_envs=8)

# PPO modeli tanımla
model = PPO(
    policy       = "MlpPolicy",
    env          = vec_env,
    learning_rate = 3e-4,
    n_steps      = 2048,     # rollout uzunluğu
    batch_size   = 64,
    n_epochs     = 10,
    gamma        = 0.99,
    gae_lambda   = 0.95,
    clip_range   = 0.2,
    ent_coef     = 0.0,
    vf_coef      = 0.5,
    max_grad_norm = 0.5,
    verbose      = 1,
    tensorboard_log = "./ppo_cartpole_tb/"
)

# Değerlendirme callback
eval_env = Monitor(gym.make("CartPole-v1"))
eval_cb  = EvalCallback(eval_env,
                         best_model_save_path="./best_model/",
                         eval_freq=5000, n_eval_episodes=20,
                         verbose=1)

# Eğit
model.learn(total_timesteps=300_000, callback=eval_cb)
model.save("ppo_cartpole_final")

# Değerlendirme
from stable_baselines3.common.evaluation import evaluate_policy
mean_r, std_r = evaluate_policy(model, eval_env, n_eval_episodes=100)
print(f"Ortalama ödül: {mean_r:.1f} ± {std_r:.1f}")

import optuna
from stable_baselines3 import PPO
from stable_baselines3.common.env_util import make_vec_env
from stable_baselines3.common.evaluation import evaluate_policy
import gymnasium as gym

def objective(trial):
    lr       = trial.suggest_float("lr", 1e-5, 1e-3, log=True)
    n_steps  = trial.suggest_categorical("n_steps", [128, 256, 512, 1024])
    clip     = trial.suggest_float("clip", 0.1, 0.4)
    gae_lam  = trial.suggest_float("gae_lambda", 0.9, 0.99)

    vec_env = make_vec_env("LunarLander-v2", n_envs=4)
    model   = PPO("MlpPolicy", vec_env, learning_rate=lr,
                   n_steps=n_steps, clip_range=clip, gae_lambda=gae_lam,
                   verbose=0)
    model.learn(100_000)

    eval_env = gym.make("LunarLander-v2")
    mean_r, _ = evaluate_policy(model, eval_env, n_eval_episodes=20)
    return mean_r

study = optuna.create_study(direction="maximize")
study.optimize(objective, n_trials=30)
print("En iyi parametreler:", study.best_params)

import wandb
from stable_baselines3 import PPO
from stable_baselines3.common.env_util import make_vec_env
from stable_baselines3.common.callbacks import BaseCallback

class WandbCallback(BaseCallback):
    def __init__(self, verbose=0):
        super().__init__(verbose)
        self.episode_rewards = []

    def _on_step(self) -> bool:
        for info in self.locals.get('infos', []):
            if 'episode' in info:
                ep_rew = info['episode']['r']
                wandb.log({
                    'episode_reward': ep_rew,
                    'timestep': self.num_timesteps
                })
        return True

# WandB başlat
wandb.init(
    project = "lunarlander-ppo",
    config  = {
        "algorithm"    : "PPO",
        "env"          : "LunarLander-v2",
        "learning_rate": 3e-4,
        "n_steps"      : 2048,
        "total_timesteps": 1_000_000
    }
)

vec_env = make_vec_env("LunarLander-v2", n_envs=8)
model   = PPO("MlpPolicy", vec_env, learning_rate=3e-4,
               n_steps=2048, verbose=0)
model.learn(total_timesteps=1_000_000, callback=WandbCallback())
wandb.finish()

import gymnasium as gym
import numpy as np
import matplotlib.pyplot as plt
from stable_baselines3 import PPO
from stable_baselines3.common.env_util import make_vec_env
from stable_baselines3.common.evaluation import evaluate_policy
from stable_baselines3.common.callbacks import EvalCallback

# Eğitim ortamı (paralel)
train_env = make_vec_env("LunarLander-v2", n_envs=8, seed=42)

# Model — SB3 varsayılan PPO ayarları iyi çalışır
model = PPO(
    "MlpPolicy", train_env,
    learning_rate = 3e-4,
    n_steps       = 1024,
    batch_size    = 64,
    n_epochs      = 4,
    gamma         = 0.999,
    gae_lambda    = 0.98,
    ent_coef      = 0.01,
    verbose       = 1,
    seed          = 42
)

# Eval callback ile en iyi modeli kaydet
eval_env = gym.make("LunarLander-v2")
eval_cb  = EvalCallback(eval_env,
                         best_model_save_path="./ll_best/",
                         eval_freq=10_000,
                         n_eval_episodes=20,
                         verbose=1)

model.learn(total_timesteps=1_000_000, callback=eval_cb)

# Son değerlendirme
mean_r, std_r = evaluate_policy(model, eval_env, n_eval_episodes=100)
print(f"Son performans: {mean_r:.1f} ± {std_r:.1f}")
if mean_r >= 200:
    print("LunarLander-v2 ÇÖZÜLDÜ!")

# Learning curve — SB3 monitor verilerini oku
from stable_baselines3.common.results_plotter import load_results, ts2xy

# Render ile izle
render_env = gym.make("LunarLander-v2", render_mode="human")
obs, _ = render_env.reset()
for _ in range(1000):
    action, _ = model.predict(obs, deterministic=True)
    obs, _, done, trunc, _ = render_env.step(action)
    if done or trunc:
        obs, _ = render_env.reset()
render_env.close()