Kernel Tracing Cheat Sheet

Frameworks

Event Tracing

Documentation

mount -t debugfs none /sys/kernel/debug

# Variant 1
echo "sched_wakeup"  >> /sys/kernel/debug/tracing/set_event
echo "!sched_wakeup" >> /sys/kernel/debug/tracing/set_event
echo "sched:*"        > /sys/kernel/debug/tracing/set_event

# Variant 2
echo 1 > /sys/kernel/debug/tracing/events/sched/sched_wakeup/enable
echo 0 > /sys/kernel/debug/tracing/events/sched/sched_wakeup/enable
echo 1 > /sys/kernel/debug/tracing/events/sched/enable

# Available events
find /sys/kernel/debug/tracing/events/sched

# To add trace points from boot loader, add
trace_event=<event name>

# Setting triggers: stacktrace, snapshot, traceon/off
# -> Always stracktrace
echo 'stacktrace' > /sys/kernel/debug/tracing/events/<event>/trigger

# -> Snapshot on next event
echo 'snapshot if nr_rq > 1' > /sys/kernel/debug/tracing/events/<event>/trigger

Perf Probe

apt-get install linux-tools-common linux-tools-generic

perf probe --line vfs_read
perf probe --add 'vfs_read file file->f_mode'
perf probe --list

# Trace Event
perf trace                         # Trace to terminal (useless because terminal feedback loop)
perf trace <command>               # Trace a command execution

perf record -e "syscalls:*" ls     # Record system calls on execution of "ls"
perf record -e "syscalls:*" -aR       # Record all processes until Ctrl-C is pressed

# Note: when "syscalls" does not exist try "raw_syscalls"

perf list                          # For list of possible -e filters

perf report -i perf.data

# Delete Event
perf probe --del '*'

Misc perf stuff

SystemTap

oprofile

statistical profiler for Linux (<20% overhead)

# After boot run "init" and "setup" with or without kernel statistics
opcontrol --init
opcontrol --setup --no-vmlinux  
opcontrol --setup --vmlinux=/usr/lib/debug/lib/modules/`uname -r`/vmlinux

opcontrol --reset
opcontrol --start
opcontrol --dump
opcontrol --shutdown

# Per library/image summaries
opreport
opreport /usr/sbin/sshd	# Limit to this binary

# Call graph for one application
opreport -cl --demangle=smart /usr/sbin/apache2

# To resolve kernel symbols
opreport --symbols --image-path=/lib/modules/<version>/kernel