Sentetik Veri Üretimi —
LLM & Diffusion.

from openai import OpenAI
import json, random, hashlib

client = OpenAI()

# ── Seed instruction'lar ──────────────────────────────────────
SEED_INSTRUCTIONS = [
    {"instruction": "Bir e-posta yaz.", "input": "", "output": "..."},
    {"instruction": "Bu metni özetle.", "input": "Uzun metin...", "output": "..."},
    {"instruction": "Bu kodun hatasını bul.", "input": "def foo(): ...", "output": "..."},
]

GENERATION_PROMPT = """Aşağıda {n_shots} örnek instruction-input-output çifti verilmiştir.
Bu formatta yeni bir tane üret. Farklı konular ve görev tipleri seç.

Örnekler:
{examples}

Şimdi yeni bir tane üret (JSON formatında):"""

def generate_instruction(seed_pool: list, n_shots: int = 3) -> dict:
    examples = json.dumps(
        random.sample(seed_pool, min(n_shots, len(seed_pool))),
        ensure_ascii=False, indent=2
    )
    prompt = GENERATION_PROMPT.format(n_shots=n_shots, examples=examples)

    response = client.chat.completions.create(
        model="gpt-4o-mini",
        messages=[{"role": "user", "content": prompt}],
        temperature=0.9,
        response_format={"type": "json_object"},
    )
    return json.loads(response.choices[0].message.content)

def quality_filter(item: dict) -> bool:
    """Basit kalite filtresi."""
    instr = item.get("instruction", "")
    if len(instr) < 10: return False
    if len(instr) > 500: return False
    if "aşağıda" in instr.lower() and "örnek" in instr.lower():
        return False  # prompt sızıntısı
    return True

def dedup_hash(item: dict) -> str:
    return hashlib.md5(item["instruction"].encode()).hexdigest()

# ── Üretim döngüsü ────────────────────────────────────────────
dataset = list(SEED_INSTRUCTIONS)
seen_hashes = {dedup_hash(x) for x in dataset}
target = 1000

while len(dataset) < target:
    try:
        new_item = generate_instruction(dataset)
        h = dedup_hash(new_item)
        if h not in seen_hashes and quality_filter(new_item):
            dataset.append(new_item)
            seen_hashes.add(h)
    except Exception as e:
        print(f"Hata: {e}")

with open("synthetic_instructions.jsonl", "w") as f:
    for item in dataset:
        f.write(json.dumps(item, ensure_ascii=False) + "\n")
print(f"Üretilen: {len(dataset)} instruction")

# pip install distilabel[openai]
from distilabel.pipeline import Pipeline
from distilabel.steps import (
    LoadDataFromDicts, KeepColumns, FilterRowsBy
)
from distilabel.steps.tasks import (
    TextGeneration, SelfInstruct, UltraFeedback
)
from distilabel.llms import OpenAILLM, vLLM

# ── Pipeline tanımı ──────────────────────────────────────────
with Pipeline(name="synthetic-qa") as pipeline:

    # 1. Seed konuları yükle
    load_topics = LoadDataFromDicts(
        data=[{"topic": t} for t in [
            "Python programlama", "Makine öğrenmesi",
            "Türk tarihi", "Matematik"
        ]],
        batch_size=4,
    )

    # 2. Her konudan instruction üret
    self_instruct = SelfInstruct(
        llm=OpenAILLM(model="gpt-4o-mini"),
        num_instructions=5,
        input_batch_size=4,
    )

    # 3. Her instruction için cevap üret
    text_gen = TextGeneration(
        llm=OpenAILLM(model="gpt-4o"),
        system_prompt="Sen uzman bir Türkçe asistansın. Kapsamlı cevaplar ver.",
        input_batch_size=8,
    )

    # 4. Kalite puanlama (UltraFeedback)
    quality_score = UltraFeedback(
        llm=OpenAILLM(model="gpt-4o"),
        aspect="overall-rating",
    )

    # 5. Düşük kalitelileri filtrele
    quality_filter = FilterRowsBy(
        filter_fn=lambda row: row["score"] >= 4,
    )

    # 6. Gereksiz sütunları temizle
    keep = KeepColumns(columns=["instruction", "generation", "score"])

    # DAG bağlantıları
    load_topics >> self_instruct >> text_gen >> quality_score >> quality_filter >> keep

# ── Pipeline çalıştırma ──────────────────────────────────────
distiset = pipeline.run(
    parameters={
        "self_instruct": {"llm": {"generation_kwargs": {"temperature": 0.9}}},
    },
    use_cache=True,
)

# ── HuggingFace'e yükle ──────────────────────────────────────
distiset.push_to_hub("emirpehlevan/synthetic-tr-qa")
print(f"Üretilen örnek: {len(distiset['train'])}")

from openai import OpenAI
import random, json

client = OpenAI()

EVOLUTION_PROMPTS = {
    "deepen": """Aşağıdaki instruction'ı daha karmaşık hale getir.
Orijinal: {instruction}
Derinleştirilmiş versiyon (daha fazla kısıt, daha spesifik): """,

    "breadth": """Aşağıdaki instruction'ın konusundan tamamen yeni bir instruction üret.
Orijinal: {instruction}
Yeni instruction (farklı ama aynı alanda): """,

    "concretize": """Aşağıdaki genel instruction'ı somut bir örneğe uygula.
Orijinal: {instruction}
Somut versiyon: """,

    "add_reasoning": """Aşağıdaki instruction'a adım adım düşünme gerektiren bir kısıt ekle.
Orijinal: {instruction}
Geliştirilmiş: """,
}

def evolve_instruction(instruction: str, strategy: str = None) -> str:
    if strategy is None:
        strategy = random.choice(list(EVOLUTION_PROMPTS.keys()))
    prompt = EVOLUTION_PROMPTS[strategy].format(instruction=instruction)

    response = client.chat.completions.create(
        model="gpt-4o-mini",
        messages=[{"role": "user", "content": prompt}],
        temperature=0.8,
    )
    return response.choices[0].message.content.strip()

def multi_round_evolution(seed_instructions: list, rounds: int = 3) -> list:
    current = list(seed_instructions)
    all_instructions = list(seed_instructions)

    for rnd in range(rounds):
        next_gen = []
        for instr in current:
            evolved = evolve_instruction(instr)
            next_gen.append(evolved)
            all_instructions.append(evolved)
        current = next_gen
        print(f"Round {rnd+1}: {len(all_instructions)} toplam instruction")

    return all_instructions

seeds = [
    "Python'da liste oluştur.",
    "Bir metin özetle.",
    "İki sayıyı karşılaştır.",
]
all_instructions = multi_round_evolution(seeds, rounds=3)
print(f"Toplam: {len(all_instructions)} instruction")

from diffusers import StableDiffusionImg2ImgPipeline
import torch
from PIL import Image
from pathlib import Path
import random

# ── Pipeline yükle ───────────────────────────────────────────
pipe = StableDiffusionImg2ImgPipeline.from_pretrained(
    "runwayml/stable-diffusion-v1-5",
    torch_dtype=torch.float16,
)
pipe = pipe.to("cuda")

# ── Augmentation varyantları ─────────────────────────────────
AUGMENTATION_PROMPTS = [
    "same scene, rainy weather, overcast sky",
    "same scene, winter, snow on the ground",
    "same scene, night time, artificial lighting",
    "same scene, golden hour, warm sunlight",
    "same scene, foggy morning, misty atmosphere",
]

def augment_image(
    img_path: str,
    n_variants: int = 3,
    strength: float = 0.5,
    guidance_scale: float = 7.5,
) -> list:
    init_image = Image.open(img_path).convert("RGB").resize((512, 512))
    prompts = random.sample(AUGMENTATION_PROMPTS, min(n_variants, len(AUGMENTATION_PROMPTS)))
    variants = []

    for prompt in prompts:
        result = pipe(
            prompt=prompt,
            image=init_image,
            strength=strength,
            guidance_scale=guidance_scale,
            num_inference_steps=30,
        ).images[0]
        variants.append((prompt, result))

    return variants

# ── Toplu augmentation ───────────────────────────────────────
input_dir  = Path("dataset/train/car")
output_dir = Path("dataset/augmented/car")
output_dir.mkdir(parents=True, exist_ok=True)

for img_path in input_dir.glob("*.jpg"):
    variants = augment_image(str(img_path), n_variants=3, strength=0.45)
    for i, (prompt, img) in enumerate(variants):
        out_path = output_dir / f"{img_path.stem}_aug{i}.jpg"
        img.save(out_path)
        print(f"Kaydedildi: {out_path}")

# pip install sdv
import pandas as pd
from sdv.metadata import SingleTableMetadata
from sdv.single_table import CTGANSynthesizer, TVAESynthesizer
from sdv.evaluation.single_table import run_diagnostic, evaluate_quality

# ── Örnek finans verisi ──────────────────────────────────────
import numpy as np
np.random.seed(42)
n = 1000
df = pd.DataFrame({
    "yas":       np.random.randint(18, 70, n),
    "gelir":     np.random.lognormal(10, 0.5, n),
    "kredi_skoru": np.clip(np.random.normal(650, 80, n), 300, 850).astype(int),
    "meslek":    np.random.choice(["mühendis", "öğretmen", "doktor", "serbest"], n),
    "kredi_onay": np.random.choice([0, 1], n, p=[0.35, 0.65]),
})

# ── Metadata tanımı ──────────────────────────────────────────
metadata = SingleTableMetadata()
metadata.detect_from_dataframe(df)
# İsteğe bağlı override:
metadata.update_column("kredi_onay", sdtype="boolean")
metadata.update_column("meslek",     sdtype="categorical")

# ── CTGAN ─────────────────────────────────────────────────────
ctgan = CTGANSynthesizer(
    metadata,
    epochs=300,
    batch_size=500,
    generator_dim=(256, 256),
    discriminator_dim=(256, 256),
    verbose=True,
)
ctgan.fit(df)
synthetic_ctgan = ctgan.sample(num_rows=5000)

# ── TVAE ──────────────────────────────────────────────────────
tvae = TVAESynthesizer(metadata, epochs=300)
tvae.fit(df)
synthetic_tvae = tvae.sample(num_rows=5000)

# ── Kalite değerlendirme ─────────────────────────────────────
diagnostic = run_diagnostic(df, synthetic_ctgan, metadata)
quality    = evaluate_quality(df, synthetic_ctgan, metadata)
print(f"Genel kalite skoru: {quality.get_score():.3f}")

# ── TSTR (Train on Synthetic, Test on Real) ──────────────────
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import roc_auc_score

features = ["yas", "gelir", "kredi_skoru"]
X_syn = synthetic_ctgan[features]; y_syn = synthetic_ctgan["kredi_onay"]
X_real = df[features];             y_real = df["kredi_onay"]

clf = RandomForestClassifier().fit(X_syn, y_syn)
auc = roc_auc_score(y_real, clf.predict_proba(X_real)[:, 1])
print(f"TSTR AUC: {auc:.3f}")  # gerçek veriye yakın olmalı

import numpy as np
from sentence_transformers import SentenceTransformer
from sklearn.metrics.pairwise import cosine_similarity
from rouge_score import rouge_scorer

# ── Embedding-based çeşitlilik skoru ─────────────────────────
model_st = SentenceTransformer("paraphrase-multilingual-MiniLM-L12-v2")

def diversity_score(texts: list) -> float:
    """Ortalama pairwise cosine dissimilarity."""
    embs = model_st.encode(texts, normalize_embeddings=True)
    sim_matrix = embs @ embs.T
    n = len(texts)
    mask = np.ones((n, n), dtype=bool)
    np.fill_diagonal(mask, False)
    avg_sim = sim_matrix[mask].mean()
    return float(1.0 - avg_sim)

def faithfulness_score(sources: list, generations: list) -> float:
    """Kaynak-üretim ortalama cosine benzerliği."""
    src_embs = model_st.encode(sources, normalize_embeddings=True)
    gen_embs = model_st.encode(generations, normalize_embeddings=True)
    sims = (src_embs * gen_embs).sum(axis=1)
    return float(sims.mean())

# ── ROUGE skoru ──────────────────────────────────────────────
scorer = rouge_scorer.RougeScorer(["rouge1", "rouge2", "rougeL"])

def batch_rouge(references: list, hypotheses: list) -> dict:
    scores = {"r1": [], "r2": [], "rL": []}
    for ref, hyp in zip(references, hypotheses):
        s = scorer.score(ref, hyp)
        scores["r1"].append(s["rouge1"].fmeasure)
        scores["r2"].append(s["rouge2"].fmeasure)
        scores["rL"].append(s["rougeL"].fmeasure)
    return {k: np.mean(v) for k, v in scores.items()}

# ── FID (görüntü) ────────────────────────────────────────────
from torchmetrics.image.fid import FrechetInceptionDistance
import torch

fid = FrechetInceptionDistance(feature=2048)
# real_imgs ve fake_imgs: (N, 3, H, W) uint8 tensor
fid.update(real_imgs, real=True)
fid.update(fake_imgs, real=False)
fid_score = fid.compute()
print(f"FID: {fid_score:.2f}")  # < 10 çok iyi, < 50 kabul edilebilir

import pandas as pd, numpy as np
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import roc_auc_score
from sklearn.model_selection import train_test_split

def membership_inference_audit(
    real_df: pd.DataFrame,
    synthetic_df: pd.DataFrame,
    feature_cols: list,
) -> float:
    """
    MIA audit: sentetik veri gerçek kayıtlarla ayırt edilebilir mi?
    AUC ~ 0.5 → iyi gizlilik; AUC ~ 1.0 → kötü gizlilik
    """
    # Gerçek: 1, sentetik: 0 etiketi
    real_sample = real_df[feature_cols].copy()
    real_sample["is_real"] = 1
    syn_sample  = synthetic_df[feature_cols].head(len(real_df)).copy()
    syn_sample["is_real"]  = 0

    combined = pd.concat([real_sample, syn_sample]).sample(frac=1)
    X = combined[feature_cols]
    y = combined["is_real"]

    X_tr, X_te, y_tr, y_te = train_test_split(X, y, test_size=0.3, random_state=42)
    clf = RandomForestClassifier(n_estimators=100).fit(X_tr, y_tr)
    auc = roc_auc_score(y_te, clf.predict_proba(X_te)[:, 1])
    return auc

features = ["yas", "gelir", "kredi_skoru"]
auc = membership_inference_audit(df, synthetic_ctgan, features)
print(f"MIA AUC: {auc:.3f}")
if auc > 0.7:
    print("⚠ Yüksek MIA riski — DP-CTGAN deneyin")
else:
    print("Gizlilik makul düzeyde")

# ── DP-CTGAN ─────────────────────────────────────────────────
from sdv.single_table import CTGANSynthesizer

dp_ctgan = CTGANSynthesizer(
    metadata,
    epochs=300,
    # DP parametreleri (deneysel)
    discriminator_dim=(256, 256),
    generator_dim=(256, 256),
    batch_size=500,
)
# Not: Tam DP garantisi için smartnoise-sdk veya opacus ile eğitim önerilir
# from snsynth import Synthesizer
# synth = Synthesizer.create("ctgan", epsilon=5.0)
# synth.fit(df, preprocessor_eps=1.0)