Hugging Face
Ekosistemi.

import os
from transformers import AutoTokenizer, AutoModel

# Opsiyonel: cache konumunu değiştir
os.environ["HF_HOME"] = "/data/hf_cache"

# İlk çağrıda Hub'dan indirir, sonraki çağrılarda cache'den yükler
model_name = "google-bert/bert-base-uncased"
tokenizer  = AutoTokenizer.from_pretrained(model_name)
model      = AutoModel.from_pretrained(model_name)

# Cache konumunu kontrol et
from huggingface_hub import scan_cache_dir
cache_info = scan_cache_dir()
for repo in cache_info.repos:
    print(f"{repo.repo_id}: {repo.size_on_disk_str}")

# Token ile private model erişimi
# export HF_TOKEN=hf_xxxx  (ortam değişkeni olarak ayarla)
private_tok = AutoTokenizer.from_pretrained(
    "my-org/private-model",
    token=os.environ.get("HF_TOKEN")
)

from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("google-bert/bert-base-uncased")

text = "Transformers are powerful."

# 1) encode: sadece input_ids döndürür
ids = tokenizer.encode(text)
print(ids)
# [101, 19081, 2024, 3928, 1012, 102]

# 2) __call__: tam batch, tüm tensorleri döndürür (önerilen)
enc = tokenizer(text, return_tensors="pt")
print(enc.keys())
# dict_keys(['input_ids', 'token_type_ids', 'attention_mask'])

print("input_ids     :", enc["input_ids"])
print("attention_mask:", enc["attention_mask"])
print("token_type_ids:", enc["token_type_ids"])

# Token isimlerini gör
tokens = tokenizer.convert_ids_to_tokens(enc["input_ids"][0].tolist())
print(tokens)
# ['[CLS]', 'transformers', 'are', 'powerful', '.', '[SEP]']

# Decode: token ID'lerini metne geri çevir
decoded = tokenizer.decode(enc["input_ids"][0], skip_special_tokens=True)
print(decoded)
# 'transformers are powerful.'

# Padding ve truncation (batch için kritik)
batch_texts = [
    "Short text.",
    "This is a much longer sentence that needs padding.",
]
batch_enc = tokenizer(
    batch_texts,
    padding=True,          # en uzun seq'e göre pad
    truncation=True,      # max_length'i aşanları kes
    max_length=512,
    return_tensors="pt",
)
print(batch_enc["input_ids"].shape)  # torch.Size([2, N])

import torch
from transformers import AutoModelForCausalLM, AutoTokenizer, AutoConfig

model_id = "mistralai/Mistral-7B-Instruct-v0.3"

# Config'i önce indir, değiştir, sonra modeli yükle
config = AutoConfig.from_pretrained(model_id)
print(config.model_type)   # mistral
print(config.hidden_size)  # 4096

# Temel yükleme parametreleri
model = AutoModelForCausalLM.from_pretrained(
    model_id,
    device_map="auto",          # GPU'lara otomatik dağıt (accelerate gerekir)
    torch_dtype=torch.bfloat16, # bf16 ile yarım bellek kullanımı
    trust_remote_code=False,   # Güvenlik: bilinmeyen kod çalıştırma
    attn_implementation="flash_attention_2",  # Flash Attention
)

print(model.get_memory_footprint() / 1e9, "GB")

from transformers import pipeline
import torch

# 1) Duygu Analizi
sentiment = pipeline(
    "sentiment-analysis",
    model="distilbert/distilbert-base-uncased-finetuned-sst-2-english",
    device=0 if torch.cuda.is_available() else -1,
)
result = sentiment("The movie was absolutely wonderful!")
print(result)
# [{'label': 'POSITIVE', 'score': 0.9998}]

# 2) NER (Named Entity Recognition)
ner = pipeline(
    "ner",
    model="dslim/bert-base-NER",
    aggregation_strategy="simple",
)
entities = ner("Hugging Face was founded in New York by Clément Delangue.")
for e in entities:
    print(f"{e['word']} → {e['entity_group']} ({e['score']:.2f})")
# Hugging Face → ORG (0.99)
# New York     → LOC (0.99)
# Clément Delangue → PER (0.98)

# 3) Metin Üretimi
generator = pipeline(
    "text-generation",
    model="openai-community/gpt2",
    device_map="auto",
)
outputs = generator(
    "Machine learning is",
    max_new_tokens=50,
    num_return_sequences=2,
    temperature=0.8,
    do_sample=True,
)
for out in outputs:
    print(out["generated_text"])
    print("---")

from datasets import load_dataset

# Hub'dan yükle
ds = load_dataset("stanfordnlp/imdb")
print(ds)
# DatasetDict({'train': Dataset({features: ['text','label'], num_rows: 25000}),
#              'test':  Dataset({features: ['text','label'], num_rows: 25000})})

# Sadece belirli split'i yükle
train_ds = load_dataset("stanfordnlp/imdb", split="train")

# Yerel CSV dosyasından yükle
local_ds = load_dataset("csv", data_files="my_data.csv")

# JSON Lines dosyasından yükle
jsonl_ds = load_dataset("json", data_files="my_data.jsonl")

# Streaming modu: büyük dataset'leri diske indirmeden işle
streamed = load_dataset("HuggingFaceFW/fineweb", streaming=True, split="train")
for sample in streamed.take(3):
    print(sample["text"][:200])

from datasets import load_dataset
from transformers import AutoTokenizer
from torch.utils.data import DataLoader

ds      = load_dataset("stanfordnlp/imdb")
tokenizer = AutoTokenizer.from_pretrained("google-bert/bert-base-uncased")

# .map(): her örneğe fonksiyon uygula
def tokenize(batch):
    return tokenizer(
        batch["text"],
        padding="max_length",
        truncation=True,
        max_length=512,
    )

tokenized = ds.map(tokenize, batched=True, num_proc=4)

# .filter(): koşulu sağlamayanları çıkar
long_reviews = ds["train"].filter(lambda x: len(x["text"].split()) > 100)

# .shuffle() ve .train_test_split()
split = ds["train"].shuffle(seed=42).train_test_split(test_size=0.1)
print(split)
# DatasetDict({'train': Dataset(22500), 'test': Dataset(2500)})

# PyTorch DataLoader'a aktar
tokenized.set_format("torch", columns=["input_ids", "attention_mask", "label"])
loader = DataLoader(tokenized["train"], batch_size=16, shuffle=True)

batch = next(iter(loader))
print(batch["input_ids"].shape)   # torch.Size([16, 512])
print(batch["label"].shape)      # torch.Size([16])

import numpy as np
from datasets import load_dataset
from transformers import (
    AutoTokenizer, AutoModelForSequenceClassification,
    TrainingArguments, Trainer, DataCollatorWithPadding,
)
import evaluate

# 1) Veri ve tokenizer
ds        = load_dataset("stanfordnlp/imdb")
tokenizer = AutoTokenizer.from_pretrained("distilbert/distilbert-base-uncased")

def preprocess(batch):
    return tokenizer(batch["text"], truncation=True, max_length=512)

tokenized = ds.map(preprocess, batched=True)

# 2) Model (2 sınıf: pozitif / negatif)
model = AutoModelForSequenceClassification.from_pretrained(
    "distilbert/distilbert-base-uncased",
    num_labels=2,
)

# 3) Metrik fonksiyonu
accuracy = evaluate.load("accuracy")

def compute_metrics(eval_pred):
    logits, labels = eval_pred
    preds = np.argmax(logits, axis=1)
    return accuracy.compute(predictions=preds, references=labels)

# 4) Eğitim argümanları
args = TrainingArguments(
    output_dir="./imdb-distilbert",
    num_train_epochs=3,
    per_device_train_batch_size=16,
    per_device_eval_batch_size=32,
    learning_rate=2e-5,
    weight_decay=0.01,
    eval_strategy="epoch",
    save_strategy="epoch",
    load_best_model_at_end=True,
    metric_for_best_model="accuracy",
    bf16=True,
    logging_steps=100,
    report_to="none",  # wandb/tensorboard devre dışı
)

# 5) Trainer oluştur ve başlat
trainer = Trainer(
    model=model,
    args=args,
    train_dataset=tokenized["train"],
    eval_dataset=tokenized["test"],
    tokenizer=tokenizer,
    data_collator=DataCollatorWithPadding(tokenizer),
    compute_metrics=compute_metrics,
)

trainer.train()
# Eğitim tamamlandıktan sonra en iyi modeli kaydet
trainer.save_model("./imdb-distilbert-best")
results = trainer.evaluate()
print(results)
# {'eval_loss': 0.18, 'eval_accuracy': 0.934, ...}

# Kurulum
pip install accelerate

# İnteraktif konfigürasyon (makine başına bir kez)
accelerate config
# Soru sorar: kaç GPU? mixed precision? DeepSpeed?
# ~/.cache/huggingface/accelerate/default_config.yaml oluşturur

# Script'i çalıştır
accelerate launch train.py

# Konfigürasyonu override ederek çalıştır
accelerate launch --num_processes 4 --mixed_precision bf16 train.py

import torch
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer
from accelerate import Accelerator

# Accelerator nesnesini oluştur — device yönetimini devralır
accelerator = Accelerator(
    mixed_precision="bf16",        # bf16 / fp16 / no
    gradient_accumulation_steps=4,
)

# Model ve tokenizer — cihaza taşımayı accelerator halleder
model_name = "distilbert/distilbert-base-uncased"
model     = AutoModelForSequenceClassification.from_pretrained(model_name, num_labels=2)
optimizer = torch.optim.AdamW(model.parameters(), lr=2e-5)

# DataLoader'ı kendiniz oluşturun
train_loader = DataLoader(train_dataset, batch_size=16, shuffle=True)

# Accelerator ile hazırla: model, optimizer, dataloader birlikte
model, optimizer, train_loader = accelerator.prepare(model, optimizer, train_loader)

# Eğitim döngüsü — .to(device) çağrısına gerek yok
model.train()
for epoch in range(3):
    for batch in train_loader:
        with accelerator.accumulate(model):
            outputs = model(**batch)
            loss    = outputs.loss
            accelerator.backward(loss)   # loss.backward() yerine
            optimizer.step()
            optimizer.zero_grad()

    if accelerator.is_main_process():
        print(f"Epoch {epoch} tamamlandı")

# Model kaydet (sadece main process)
accelerator.wait_for_everyone()
unwrapped = accelerator.unwrap_model(model)
unwrapped.save_pretrained("./output", is_main_process=accelerator.is_main_process())

import torch
from transformers import AutoModelForCausalLM, AutoTokenizer, TrainingArguments, Trainer
from peft import LoraConfig, get_peft_model, TaskType, PeftModel
from datasets import load_dataset

model_id  = "mistralai/Mistral-7B-v0.3"
tokenizer = AutoTokenizer.from_pretrained(model_id)
tokenizer.pad_token = tokenizer.eos_token

# Modeli bf16 ile yükle
model = AutoModelForCausalLM.from_pretrained(
    model_id,
    torch_dtype=torch.bfloat16,
    device_map="auto",
)

# LoRA konfigürasyonu
lora_config = LoraConfig(
    task_type=TaskType.CAUSAL_LM,
    r=16,                   # rank — küçük = az parametre
    lora_alpha=32,           # scaling faktörü
    lora_dropout=0.05,       # adaptör dropout
    target_modules=[         # hangi katmanlar eğitilecek
        "q_proj", "k_proj", "v_proj",
        "o_proj", "gate_proj", "up_proj", "down_proj",
    ],
    bias="none",
)

# PEFT modelini oluştur
peft_model = get_peft_model(model, lora_config)
peft_model.print_trainable_parameters()
# trainable params: 20,185,088 || all params: 7,261,577,216 || trainable%: 0.2780

# Normal Trainer ile eğit
args = TrainingArguments(
    output_dir="./mistral-lora",
    num_train_epochs=2,
    per_device_train_batch_size=2,
    gradient_accumulation_steps=8,
    learning_rate=2e-4,
    bf16=True,
    logging_steps=50,
    save_strategy="epoch",
    report_to="none",
)
trainer = Trainer(model=peft_model, args=args, train_dataset=train_ds)
trainer.train()

# Adaptörü kaydet (sadece LoRA ağırlıkları — küçük dosya)
peft_model.save_pretrained("./mistral-lora-adapter")

# Daha sonra yükle ve birleştir
base_model = AutoModelForCausalLM.from_pretrained(model_id, torch_dtype=torch.bfloat16)
merged     = PeftModel.from_pretrained(base_model, "./mistral-lora-adapter")
merged     = merged.merge_and_unload()  # LoRA ağırlıklarını ana modele gömülü hale getir
merged.save_pretrained("./mistral-merged")

import torch
from safetensors.torch import save_file, load_file
from transformers import AutoModelForSequenceClassification, AutoTokenizer

model_id = "distilbert/distilbert-base-uncased"
model    = AutoModelForSequenceClassification.from_pretrained(model_id, num_labels=2)

# safetensors ile manuel kaydet
state_dict = {k: v.cpu() for k, v in model.state_dict().items()}
save_file(state_dict, "model.safetensors")

# Geri yükle
loaded_state = load_file("model.safetensors")
model.load_state_dict(loaded_state)

# save_pretrained zaten safetensors kullanır (transformers >= 4.34)
model.save_pretrained("./my-model")       # model.safetensors oluşturur
tokenizer.save_pretrained("./my-model")   # tokenizer.json vb.

# ─── Hub'a Yükle ─────────────────────────────────────────
from huggingface_hub import HfApi, create_repo

api = HfApi()

# Repo oluştur (yoksa)
create_repo("my-username/imdb-distilbert", exist_ok=True)

# Klasörü doğrudan Hub'a yükle
api.upload_folder(
    folder_path="./my-model",
    repo_id="my-username/imdb-distilbert",
    repo_type="model",
)

# Alternatif: push_to_hub ile doğrudan gönder
model.push_to_hub("my-username/imdb-distilbert")
tokenizer.push_to_hub("my-username/imdb-distilbert")

# ─── Model Card (README.md) ──────────────────────────────
from huggingface_hub import ModelCard

card_content = """---
language: en
license: apache-2.0
tags:
  - text-classification
  - sentiment-analysis
datasets:
  - stanfordnlp/imdb
metrics:
  - accuracy
---
# IMDB Sentiment Classifier
DistilBERT fine-tuned on IMDB for binary sentiment analysis.
Accuracy: 93.4% on test set.
"""
card = ModelCard(card_content)
card.push_to_hub("my-username/imdb-distilbert")