strace · perf · valgrind —
Linux Debugging Araçları.

# Debian / Ubuntu
sudo apt install strace ltrace linux-perf valgrind

# RHEL / CentOS / Fedora
sudo yum install strace ltrace perf valgrind

# perf kernel sürümü doğrula
perf --version
uname -r

# perf paranoia seviyesini düşür (root olmadan kullanmak için)
sudo sysctl kernel.perf_event_paranoid=1

strace ls /tmp 2>&1 | head -20

execve("/usr/bin/ls", ["ls", "/tmp"], 0x... /* env vars */) = 0
brk(NULL)                               = 0x55f8c2a01000
openat(AT_FDCWD, "/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=89432, ...}) = 0
mmap(NULL, 89432, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f3a2c1d0000
close(3)                                = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3...", 832)       = 832
mmap(NULL, 2037344, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f3a...
close(3)                                = 0
getdents64(3, /* 12 entries */, 32768) = 336
write(1, "file1.txt  file2.log\n", 21)   = 21
close(1)                                = 0
exit_group(0)                           = ?

strace -c ls /tmp

% time     seconds  usecs/call     calls    errors syscall
------ ----------- ----------- --------- --------- ----------------
 38.21    0.000412          41        10           mmap
 22.14    0.000239          29         8           openat
 14.87    0.000160          22         7           read
  8.92    0.000096          13         7           fstat
  5.43    0.000058          14         4           close
  4.11    0.000044          11         4           mprotect
  3.72    0.000040          13         3           brk
  2.60    0.000028          28         1           getdents64
------ ----------- ----------- --------- --------- ----------------
100.00    0.001077                    44         0 total

# Sadece network syscall'ları, süre ile
strace -T -e trace=network curl https://example.com 2>&1 | head -15

# bash + fork + exec zincirini takip et
strace -f bash -c "ls /tmp && cat /etc/hostname"

# Her satırın başına PID ekle (çok process varsa şart)
strace -f -e trace=execve -o /tmp/trace.txt ./myapp

# PID bul
pidof nginx

# Çalışan process'e attach
strace -p 1234

# Tüm nginx worker'larını izle (-f ile birlikte)
strace -f -p 1234 -e trace=network 2>&1 | head -50

# Çıktıyı dosyaya yaz (stderr yerine)
strace -o /tmp/strace.log ./myapp

# -f ile çok process: her process ayrı dosyaya (PID eklenir)
strace -f -o /tmp/trace ./myapp
# → /tmp/trace.1234, /tmp/trace.1235, ...

# Default: 32 karakter (path'ler kesilir)
strace ./myapp
# openat(AT_FDCWD, "/etc/myapp/config/application.co"..., O_RDONLY) = -1

# -s 256 ile tam path
strace -s 256 ./myapp
# openat(AT_FDCWD, "/etc/myapp/config/application.conf", O_RDONLY) = -1 ENOENT

# -y: fd numarasını dosya adına çevir
strace -y ./myapp
# read(3</etc/passwd>, ...)  — fd 3'ün /etc/passwd olduğu görünür

# -yy: socket fd'lerini adres:port olarak göster
strace -yy ./myapp
# sendto(5<TCP:127.0.0.1:8080->10.0.0.1:443>, ...)

# 1. Sadece dosya açma hataları filtrele
strace -e trace=openat,open -s 256 ./myapp 2>&1 \
  | grep ENOENT

# Çıktı:
# openat(AT_FDCWD, "/etc/myapp.conf", O_RDONLY) = -1 ENOENT
# openat(AT_FDCWD, "/home/user/.config/myapp.conf", O_RDONLY) = -1 ENOENT
# → Program /etc/myapp.conf'u arıyor, dosya mevcut değil

# 2. Çözüm: doğru path'e semlink veya dosya oluştur
sudo cp myapp.conf.example /etc/myapp.conf

ltrace ls /tmp 2>&1 | head -20

__libc_start_main(0x401b20, 2, 0x7ffd..., ...)              = 0
setlocale(LC_ALL, "")                                      = "en_US.UTF-8"
bindtextdomain("coreutils", "/usr/share/locale")           = "/usr/share/locale"
textdomain("coreutils")                                    = "coreutils"
__cxa_atexit(0x40a480, 0, 0x603260)                         = 0
isatty(1)                                                   = 1
malloc(40)                                                  = 0x604670
opendir("/tmp")                                             = 0x6046b0
readdir(0x6046b0)                                           = 0x6046f0
free(0x604670)                                              = <void>

# Özet istatistik
ltrace -c ./myapp

# Sadece malloc ve free çağrıları
ltrace -e malloc+free ./myapp

# Belirli bir kütüphaneyi izle
ltrace -l libssl.so.3 openssl s_client -connect example.com:443

# Child process'leri de takip et
ltrace -f ./myapp

perf stat ./myapp

 Performance counter stats for './myapp':

          1,842.53 msec task-clock         #    0.998 CPUs utilized
                 4      context-switches   #    2.171 /sec
                 0      cpu-migrations     #    0.000 /sec
               512      page-faults        #  277.857 /sec

     4,921,034,128      cycles             #    2.671 GHz
     6,284,019,442      instructions       #    1.28  insn per cycle  ← IPC
       847,234,891      branches           #  459.788 M/sec
         3,421,023      branch-misses      #    0.40% of all branches

       1.845761234 seconds time elapsed
       1.840123000 seconds user
       0.004534000 seconds sys

# Cache miss detayı
perf stat -e cache-misses,cache-references,LLC-loads,LLC-load-misses ./myapp

# Mevcut event listesi
perf list | grep cache

# 5 kez çalıştır, ortalama ve standart sapma hesapla
perf stat -r 5 ./myapp

       1.847213534 seconds time elapsed   ( +-  0.23% )
       1.841890200 seconds user            ( +-  0.24% )
       0.005230600 seconds sys             ( +-  1.12% )

# Tüm sistem genelinde CPU yiyen fonksiyonlar (htop gibi ama fonksiyon bazlı)
sudo perf top

# Belirli bir process
sudo perf top -p $(pidof myapp)

# Call-graph ile (hangi fonksiyon kimi çağırıyor)
sudo perf top -g

# -g: call-graph (backtrace) kaydet — flamegraph için şart
perf record -g ./myapp

# Çalışan process'e attach
sudo perf record -g -p $(pidof myapp) -- sleep 10

# Frekans ayarla (Hz): default 4000, daha az overhead için düşür
perf record -F 99 -g ./myapp

# → perf.data dosyası oluşturulur
ls -lh perf.data

# İnteraktif TUI (ok tuşlarıyla gezin, Enter ile aç)
perf report

# Text çıktı (pipe veya CI için)
perf report --stdio 2>&1 | head -40

# Belirli bir sembol için assembly annotation
perf annotate main --stdio

# Overhead  Command   Shared Object     Symbol
# ........  ........  ................  ......................
#
    42.31%  myapp     myapp             [.] process_buffer
    18.74%  myapp     libc.so.6         [.] __memcpy_avx_unaligned
     9.12%  myapp     myapp             [.] parse_header
     7.03%  myapp     myapp             [.] hash_lookup
     4.21%  myapp     [kernel]          [k] __get_user_pages

# Brendan Gregg'in FlameGraph araçlarını indir
git clone https://github.com/brendangregg/FlameGraph /opt/flamegraph

# Profil kaydet
perf record -F 99 -g -p $(pidof myapp) -- sleep 30

# SVG üret
perf script \
  | /opt/flamegraph/stackcollapse-perf.pl \
  | /opt/flamegraph/flamegraph.pl > /tmp/flame.svg

# Tarayıcıda aç
xdg-open /tmp/flame.svg

# Tam leak analizi
valgrind --leak-check=full --show-leak-kinds=all \
         --track-origins=yes -v ./myapp 2>&1 | tee valgrind.log

#include <stdlib.h>
#include <string.h>
#include <stdio.h>

int main(void) {
    char *buf = malloc(64);
    strcpy(buf, "merhaba");

    free(buf);          /* buf serbest bırakıldı */

    /* HATA: free sonrası erişim — use-after-free */
    printf("%s\n", buf);

    return 0;
}

==12345== Invalid read of size 1
==12345==    at 0x4C33A22: strlen (vg_replace_strmem.c:459)
==12345==    at 0x1091C5: main (uaf-bug.c:12)
==12345==  Address 0x5204040 is 0 bytes inside a block of size 64 free'd
==12345==    at 0x4C30D3B: free (vg_replace_malloc.c:530)
==12345==    at 0x1091BE: main (uaf-bug.c:9)
==12345==  Block was alloc'd at
==12345==    at 0x4C2FB0F: malloc (vg_replace_malloc.c:309)
==12345==    at 0x10918B: main (uaf-bug.c:6)
==12345==
==12345== LEAK SUMMARY:
==12345==    definitely lost: 0 bytes in 0 blocks
==12345==    indirectly lost: 0 bytes in 0 blocks
==12345==    possibly lost:   0 bytes in 0 blocks
==12345==    still reachable: 0 bytes in 0 blocks

{
   openssl_false_positive
   Memcheck:Leak
   match-leak-kinds: reachable
   fun:malloc
   obj:*libcrypto*
}

valgrind --suppressions=my.supp --leak-check=full ./myapp

# ASan + sembol için -g şart
gcc -fsanitize=address -g -O1 uaf-bug.c -o myapp-asan

# Derleme sonrası normal gibi çalıştır
./myapp-asan

==12346==ERROR: AddressSanitizer: heap-use-after-free on address 0x602000000010
READ of size 1 at 0x602000000010 thread T0
    #0 0x401234 in strlen /usr/include/string.h:395
    #1 0x401456 in main uaf-bug.c:12
    #2 0x7f3a2b main in libc-start.c

0x602000000010 is located 0 bytes inside of 64-byte region
FREED by thread T0 here:
    #0 0x7f3a2b in __interceptor_free sanitizer_common_interceptors.inc:752
    #1 0x4013e2 in main uaf-bug.c:9

ALLOCATED by thread T0 here:
    #0 0x7f3a2b in __interceptor_malloc sanitizer_common_interceptors.inc:823
    #1 0x401378 in main uaf-bug.c:6

Shadow bytes around the buggy address:
  0x0c047fff7fb0: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa
  0x0c047fff7fc0: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa
=>0x0c047fff7fd0:[fd]fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd

# ASan + UBSan kombinasyonu (yaygın CI konfigürasyonu)
gcc -fsanitize=address,undefined -g -O1 myapp.c -o myapp-san

# TSan ayrı derlenmeli
gcc -fsanitize=thread -g -O1 myapp.c -o myapp-tsan

# 1. Shared library bağımlılıkları kontrol
ldd ./myapp
# → "libmylib.so.2 => not found"  varsa sorun bu

# 2. strace ile tam başlangıç izle
strace -e trace=openat,open,read -s 256 ./myapp 2>&1 | grep -E "ENOENT|EACCES"

# Tipik çıktı:
# openat(AT_FDCWD, "/etc/myapp/myapp.conf", O_RDONLY) = -1 ENOENT
# → Config dosyası eksik

# 3. execve başarısız oluyorsa
strace -e trace=execve ./myapp 2>&1
# execve("/usr/bin/myapp", ...) = -1 EACCES  → execute izni yok
chmod +x ./myapp

# 1. Hızlı kontrol: RSS artışını izle
while true; do
    cat /proc/$(pidof myapp)/status | grep VmRSS
    sleep 5
done

# 2. smaps ile hangi mapping büyüyor?
cat /proc/$(pidof myapp)/smaps | grep -A 1 heap

# 3. valgrind ile leak tespiti (test ortamında)
valgrind --leak-check=full --track-origins=yes \
         --log-file=leak.log ./myapp &
sleep 60
kill %1
grep "definitely lost" leak.log

# 1. Hangi fonksiyon CPU yiyor? (canlı)
sudo perf top -p $(pidof myapp)

# 2. 30 saniye profil al
sudo perf record -F 99 -g -p $(pidof myapp) -- sleep 30

# 3. Flamegraph üret
perf script | /opt/flamegraph/stackcollapse-perf.pl \
             | /opt/flamegraph/flamegraph.pl > /tmp/cpu-flame.svg

# 4. Text rapor (en pahalı 10 fonksiyon)
perf report --stdio --sort overhead | head -20

# 1. Core dump aktif et
ulimit -c unlimited
echo "/tmp/core.%p" | sudo tee /proc/sys/kernel/core_pattern

# 2. Uygulamayı çalıştır (crash olsun)
./myapp
# Segmentation fault (core dumped)

# 3. gdb ile core analiz
gdb ./myapp /tmp/core.$(pidof myapp 2>/dev/null || echo "*")
# gdb prompt:
# (gdb) bt            ← backtrace
# (gdb) bt full       ← tüm yerel değişkenler
# (gdb) info locals   ← mevcut frame değişkenleri
# (gdb) frame 2       ← call stack'te 2. frame'e git

# 4. Alternatif: ASan ile yeniden derle (stack trace daha açık)
gcc -fsanitize=address -g -O1 myapp.c -o myapp-asan && ./myapp-asan