Multi-Agent RL
MARL & PettingZoo.

import numpy as np

class SimpleCoopEnv:
    """2 ajan, ortak grid hedefi. Ödül: +10 hedefe, -0.1 her adım."""
    def __init__(self, size=5):
        self.size  = size
        self.moves = {0:(0,1), 1:(0,-1), 2:(1,0), 3:(-1,0)}

    def reset(self):
        self.pos  = [np.array([0,0]), np.array([0,1])]
        self.goal = np.array([self.size-1, self.size-1])
        return [np.concatenate([p, self.goal]) for p in self.pos]

    def step(self, actions):
        for i, a in enumerate(actions):
            self.pos[i] = np.clip(
                self.pos[i] + np.array(self.moves[a]), 0, self.size-1)
        done = all(np.array_equal(p, self.goal) for p in self.pos)
        r    = 10.0 if done else -0.1
        return [np.concatenate([p, self.goal]) for p in self.pos], [r,r], done

import torch, torch.nn as nn

class QMIXMixer(nn.Module):
    def __init__(self, n_agents, state_dim, embed=32):
        super().__init__()
        self.n = n_agents; self.e = embed
        self.hw1 = nn.Sequential(
            nn.Linear(state_dim, embed), nn.ReLU(),
            nn.Linear(embed, n_agents * embed))
        self.hw2 = nn.Sequential(
            nn.Linear(state_dim, embed), nn.ReLU(),
            nn.Linear(embed, embed))
        self.hb1 = nn.Linear(state_dim, embed)
        self.hb2 = nn.Sequential(
            nn.Linear(state_dim, embed), nn.ReLU(), nn.Linear(embed, 1))

    def forward(self, qs, s):
        # qs: (B, n_agents)  s: (B, state_dim)
        bs = qs.shape[0]
        w1 = torch.abs(self.hw1(s)).view(bs, self.n, self.e)
        b1 = self.hb1(s).view(bs, 1, self.e)
        h  = torch.nn.functional.elu(
            torch.bmm(qs.unsqueeze(1), w1) + b1)
        w2 = torch.abs(self.hw2(s)).view(bs, self.e, 1)
        b2 = self.hb2(s).view(bs, 1, 1)
        return (torch.bmm(h, w2) + b2).squeeze()

import torch, torch.nn as nn

class Actor(nn.Module):
    def __init__(self, obs_dim, act_dim, h=128):
        super().__init__()
        self.net = nn.Sequential(
            nn.Linear(obs_dim, h), nn.ReLU(),
            nn.Linear(h, h),       nn.ReLU(),
            nn.Linear(h, act_dim), nn.Tanh())
    def forward(self, x): return self.net(x)

class CentralCritic(nn.Module):
    def __init__(self, total_obs, total_act, h=128):
        super().__init__()
        self.net = nn.Sequential(
            nn.Linear(total_obs + total_act, h), nn.ReLU(),
            nn.Linear(h, h), nn.ReLU(),
            nn.Linear(h, 1))
    def forward(self, obs, acts):
        return self.net(torch.cat([obs, acts], dim=-1)).squeeze(-1)

def actor_loss(i, actors, critic, batch, N):
    """Ajan i için MADDPG policy kaybı."""
    obs  = [batch[f"o{j}"] for j in range(N)]
    acts = [actors[j](obs[j]) if j == i
            else actors[j](obs[j]).detach()
            for j in range(N)]
    return -critic(
        torch.cat(obs, -1), torch.cat(acts, -1)).mean()

import random, copy

class SelfPlayPool:
    """Geçmiş policy snapshot havuzu. Stratejik cycling'i azaltır."""
    def __init__(self, model, max_size=20, save_every=1000):
        self.pool       = [copy.deepcopy(model).eval()]
        self.max_size   = max_size
        self.save_every = save_every
        self.steps      = 0

    def step(self, model):
        self.steps += 1
        if self.steps % self.save_every == 0:
            snap = copy.deepcopy(model).eval()
            if len(self.pool) >= self.max_size: self.pool.pop(0)
            self.pool.append(snap)

    def opponent(self, current, p_latest=0.5):
        if random.random() < p_latest: return current
        return random.choice(self.pool)

from pettingzoo.classic import chess_v6
from pettingzoo.butterfly import cooperative_pong_v5

# AEC API — satranç (sıra tabanlı)
env = chess_v6.env()
env.reset(seed=42)
for agent in env.agent_iter():
    obs, r, term, trunc, info = env.last()
    if term or trunc:
        action = None
    else:
        action = env.action_space(agent).sample(
            mask=obs["action_mask"])
    env.step(action)
env.close()

# Parallel API — cooperative pong (eş zamanlı)
env = cooperative_pong_v5.parallel_env()
obs, _ = env.reset(seed=42)
while env.agents:
    actions = {a: env.action_space(a).sample() for a in env.agents}
    obs, rews, terms, truncs, _ = env.step(actions)
env.close()

import torch, torch.nn as nn

class CommNetStep(nn.Module):
    """
    Her ajana diğerlerinin hidden ortalamasını mesaj olarak iletir.
    hidden_states shape: (n_agents, batch, hidden_dim)
    """
    def __init__(self, h):
        super().__init__()
        self.enc = nn.Linear(h, h)
        self.com = nn.Linear(h * 2, h)

    def forward(self, hs):
        n    = hs.shape[0]
        msgs = self.enc(hs)
        mean = (msgs.sum(0, keepdim=True) - msgs) / (n - 1 + 1e-8)
        return torch.tanh(self.com(torch.cat([hs, mean], dim=-1)))

pip install "pettingzoo[classic]" chess torch

import torch, torch.nn as nn
import numpy as np
import copy, random
from pettingzoo.classic import chess_v6

class ChessNet(nn.Module):
    def __init__(self, obs_shape, n_act):
        super().__init__()
        flat = int(np.prod(obs_shape))
        self.net = nn.Sequential(
            nn.Flatten(),
            nn.Linear(flat, 512), nn.ReLU(),
            nn.Linear(512, 256),  nn.ReLU(),
            nn.Linear(256, n_act))

    def act(self, obs_t, mask):
        logits = self.net(obs_t)
        logits[~torch.BoolTensor(mask)] = -1e9
        return logits.argmax().item()


def play_game(env, p0, p1):
    env.reset()
    pols   = {"player_0": p0, "player_1": p1}
    totals = {"player_0": 0.0, "player_1": 0.0}
    for ag in env.agent_iter():
        obs, r, done, trunc, _ = env.last()
        totals[ag] += r
        if done or trunc:
            env.step(None); continue
        t = torch.FloatTensor(obs["observation"]).unsqueeze(0)
        env.step(pols[ag].act(t, obs["action_mask"]))
    return totals


def train(n_ep=300, pool_max=10, save_every=25):
    env    = chess_v6.env()
    env.reset()
    obs_sh = env.observation_space("player_0")["observation"].shape
    n_act  = env.action_space("player_0").n
    model  = ChessNet(obs_sh, n_act)
    pool   = [copy.deepcopy(model).eval()]
    w = l = d = 0

    for ep in range(n_ep):
        opp    = random.choice(pool)
        key    = random.choice(["player_0", "player_1"])
        p0, p1 = (model, opp) if key == "player_0" else (opp, model)
        res    = play_game(env, p0, p1)
        r      = res[key]
        if   r > 0: w += 1
        elif r < 0: l += 1
        else:        d += 1

        if (ep + 1) % save_every == 0:
            if len(pool) >= pool_max: pool.pop(0)
            pool.append(copy.deepcopy(model).eval())
            tot = w + l + d
            print(f"[{ep+1:3d}] W:{w} L:{l} D:{d}  WR:{w/max(tot,1):.1%}")

    env.close()
    torch.save(model.state_dict(), "chess_sp.pt")
    print("Kaydedildi: chess_sp.pt")

if __name__ == "__main__":
    train()