0.2.0: config file, auto-prune, notify-send, cgroup PSI, kernel stacks, bpftrace, tray tooltip
This commit is contained in:
luna
parent
d41f861802
commit
41dcb0f577
7 changed files with 206 additions and 39 deletions
|
|
@ -2,23 +2,116 @@
|
|||
# Continuously samples system state. Writes a detailed snapshot when
|
||||
# anomaly thresholds are crossed (D-states, PSI pressure, blocked procs).
|
||||
|
||||
set -u
|
||||
|
||||
LOG_DIR=/var/log/freeze-watcher
|
||||
CONFIG=/etc/freeze-watcher.conf
|
||||
|
||||
# --- defaults (overridden by /etc/freeze-watcher.conf) -------------------
|
||||
PSI_IO_THRESHOLD=40
|
||||
PSI_CPU_THRESHOLD=50
|
||||
DSTATE_THRESHOLD=5
|
||||
BLOCKED_THRESHOLD=5
|
||||
LOAD_THRESHOLD=20
|
||||
COOLDOWN=30
|
||||
SAMPLE_INTERVAL=1
|
||||
MAX_SNAPSHOTS=200
|
||||
MAX_SNAPSHOT_AGE_DAYS=30
|
||||
NOTIFY_USERS=""
|
||||
NOTIFY_URGENCY=normal
|
||||
CAPTURE_KERNEL_STACKS=1
|
||||
CAPTURE_BPFTRACE=0
|
||||
CAPTURE_CGROUP_PSI=1
|
||||
|
||||
[ -f "$CONFIG" ] && . "$CONFIG"
|
||||
|
||||
mkdir -p "$LOG_DIR"
|
||||
|
||||
# Thresholds
|
||||
PSI_IO_THRESHOLD=40 # io pressure avg10 % to trigger
|
||||
PSI_CPU_THRESHOLD=50 # cpu pressure avg10 % to trigger
|
||||
DSTATE_THRESHOLD=5 # processes in D state
|
||||
BLOCKED_THRESHOLD=5 # procs_blocked from /proc/stat
|
||||
LOAD_THRESHOLD=20 # 1-min load average
|
||||
COOLDOWN=30 # seconds between captures
|
||||
SAMPLE_INTERVAL=1 # seconds between samples
|
||||
|
||||
last_capture=0
|
||||
captures_since_prune=0
|
||||
|
||||
# Send a desktop notification to each configured user.
|
||||
# Daemon runs as root; we sudo into the user's session for DBus access.
|
||||
notify_users() {
|
||||
[ -z "$NOTIFY_USERS" ] && return 0
|
||||
local body="$1"
|
||||
local IFS=,
|
||||
for user in $NOTIFY_USERS; do
|
||||
user=$(echo "$user" | tr -d '[:space:]')
|
||||
[ -z "$user" ] && continue
|
||||
local uid
|
||||
uid=$(id -u "$user" 2>/dev/null) || continue
|
||||
sudo -u "$user" \
|
||||
DBUS_SESSION_BUS_ADDRESS="unix:path=/run/user/$uid/bus" \
|
||||
notify-send -u "$NOTIFY_URGENCY" -a "freeze-watcher" \
|
||||
"Freeze captured" "$body" 2>/dev/null &
|
||||
done
|
||||
}
|
||||
|
||||
# Delete old snapshots based on MAX_SNAPSHOTS and MAX_SNAPSHOT_AGE_DAYS.
|
||||
prune_snapshots() {
|
||||
if [ "${MAX_SNAPSHOT_AGE_DAYS:-0}" -gt 0 ]; then
|
||||
find "$LOG_DIR" -maxdepth 1 -name 'snap-*.log' -type f \
|
||||
-mtime "+$MAX_SNAPSHOT_AGE_DAYS" -delete 2>/dev/null
|
||||
fi
|
||||
if [ "${MAX_SNAPSHOTS:-0}" -gt 0 ]; then
|
||||
local files
|
||||
files=$(ls -1t "$LOG_DIR"/snap-*.log 2>/dev/null | tail -n "+$((MAX_SNAPSHOTS+1))")
|
||||
[ -n "$files" ] && echo "$files" | xargs -r rm -f
|
||||
fi
|
||||
}
|
||||
|
||||
capture_kernel_stacks() {
|
||||
echo "## Kernel stack traces of D-state procs"
|
||||
ps -eo pid,state --no-headers | awk '$2=="D"{print $1}' | while read -r pid; do
|
||||
local comm
|
||||
comm=$(cat "/proc/$pid/comm" 2>/dev/null) || continue
|
||||
echo "--- pid=$pid comm=$comm ---"
|
||||
cat "/proc/$pid/stack" 2>/dev/null | head -20
|
||||
done
|
||||
}
|
||||
|
||||
capture_cgroup_psi() {
|
||||
echo "## Top cgroups by PSI io.some avg10"
|
||||
{
|
||||
for f in /sys/fs/cgroup/io.pressure \
|
||||
/sys/fs/cgroup/*/io.pressure \
|
||||
/sys/fs/cgroup/*/*/io.pressure; do
|
||||
[ -r "$f" ] || continue
|
||||
local v
|
||||
v=$(awk '/^some/{for(i=1;i<=NF;i++) if($i~/^avg10=/){gsub(/avg10=/,"",$i); print $i; exit}}' "$f" 2>/dev/null)
|
||||
[ -z "$v" ] && continue
|
||||
[ "$v" = "0.00" ] && continue
|
||||
printf "%s\t%s\n" "$v" "$(dirname "$f" | sed 's|/sys/fs/cgroup||')"
|
||||
done
|
||||
} 2>/dev/null | sort -rn | head -10
|
||||
echo
|
||||
echo "## Top cgroups by PSI cpu.some avg10"
|
||||
{
|
||||
for f in /sys/fs/cgroup/cpu.pressure \
|
||||
/sys/fs/cgroup/*/cpu.pressure \
|
||||
/sys/fs/cgroup/*/*/cpu.pressure; do
|
||||
[ -r "$f" ] || continue
|
||||
local v
|
||||
v=$(awk '/^some/{for(i=1;i<=NF;i++) if($i~/^avg10=/){gsub(/avg10=/,"",$i); print $i; exit}}' "$f" 2>/dev/null)
|
||||
[ -z "$v" ] && continue
|
||||
[ "$v" = "0.00" ] && continue
|
||||
printf "%s\t%s\n" "$v" "$(dirname "$f" | sed 's|/sys/fs/cgroup||')"
|
||||
done
|
||||
} 2>/dev/null | sort -rn | head -10
|
||||
}
|
||||
|
||||
capture_bpftrace() {
|
||||
command -v bpftrace >/dev/null 2>&1 || { echo "bpftrace not installed"; return; }
|
||||
echo "## bpftrace: top kernel stacks for io_schedule waits over 2s"
|
||||
timeout 3 bpftrace -e '
|
||||
kprobe:io_schedule { @[kstack(5)] = count(); }
|
||||
interval:s:2 { exit(); }
|
||||
' 2>/dev/null | tail -40
|
||||
}
|
||||
|
||||
capture_snapshot() {
|
||||
local reason="$1"
|
||||
local now=$(date +%s)
|
||||
local file="$LOG_DIR/snap-$(date +%Y%m%d-%H%M%S).log"
|
||||
{
|
||||
echo "=== FREEZE EVENT TRIGGERED: $reason ==="
|
||||
|
|
@ -35,9 +128,17 @@ capture_snapshot() {
|
|||
echo "[$k]"; cat /proc/pressure/$k 2>/dev/null
|
||||
done
|
||||
echo
|
||||
echo "## D-state processes (with kernel wchan = where they're stuck)"
|
||||
if [ "${CAPTURE_CGROUP_PSI:-1}" = "1" ]; then
|
||||
capture_cgroup_psi
|
||||
echo
|
||||
fi
|
||||
echo "## D-state processes (with kernel wchan)"
|
||||
ps -eo state,pid,user,wchan:25,comm,args --no-headers | awk '$1=="D"' | head -30
|
||||
echo
|
||||
if [ "${CAPTURE_KERNEL_STACKS:-1}" = "1" ]; then
|
||||
capture_kernel_stacks
|
||||
echo
|
||||
fi
|
||||
echo "## Top 15 by CPU"
|
||||
ps -eo pid,user,pcpu,pmem,state,comm --no-headers --sort=-pcpu | head -15
|
||||
echo
|
||||
|
|
@ -56,6 +157,10 @@ capture_snapshot() {
|
|||
echo "## softirq"
|
||||
cat /proc/softirqs
|
||||
echo
|
||||
if [ "${CAPTURE_BPFTRACE:-0}" = "1" ]; then
|
||||
capture_bpftrace
|
||||
echo
|
||||
fi
|
||||
echo "## Recent dmesg (last 50 lines)"
|
||||
dmesg -T | tail -50
|
||||
echo
|
||||
|
|
@ -63,24 +168,19 @@ capture_snapshot() {
|
|||
journalctl --since "60 seconds ago" -p warning --no-pager 2>&1 | tail -40
|
||||
} > "$file" 2>&1
|
||||
echo "$(date -Iseconds) Captured snapshot to $file (reason: $reason)" >> "$LOG_DIR/events.log"
|
||||
notify_users "$reason"
|
||||
}
|
||||
|
||||
echo "$(date -Iseconds) freeze-watcher started (pid $$)" >> "$LOG_DIR/events.log"
|
||||
prune_snapshots
|
||||
|
||||
while true; do
|
||||
now=$(date +%s)
|
||||
|
||||
# PSI pressure - parse "some avg10=X.XX"
|
||||
psi_io=$(awk '/^some/{for(i=1;i<=NF;i++) if($i~/^avg10=/){gsub(/avg10=/,"",$i); print int($i+0.5); exit}}' /proc/pressure/io 2>/dev/null)
|
||||
psi_cpu=$(awk '/^some/{for(i=1;i<=NF;i++) if($i~/^avg10=/){gsub(/avg10=/,"",$i); print int($i+0.5); exit}}' /proc/pressure/cpu 2>/dev/null)
|
||||
|
||||
# D-state count
|
||||
dstate=$(ps -eo state --no-headers | grep -c "^D")
|
||||
|
||||
# /proc/stat blocked
|
||||
blocked=$(awk '/^procs_blocked/{print $2}' /proc/stat)
|
||||
|
||||
# Load average
|
||||
load1=$(awk '{print int($1+0.5)}' /proc/loadavg)
|
||||
|
||||
triggered=""
|
||||
|
|
@ -93,6 +193,11 @@ while true; do
|
|||
if [ -n "$triggered" ] && [ $((now - last_capture)) -ge "$COOLDOWN" ]; then
|
||||
capture_snapshot "$triggered" &
|
||||
last_capture=$now
|
||||
captures_since_prune=$((captures_since_prune+1))
|
||||
if [ "$captures_since_prune" -ge 20 ]; then
|
||||
prune_snapshots &
|
||||
captures_since_prune=0
|
||||
fi
|
||||
fi
|
||||
|
||||
sleep "$SAMPLE_INTERVAL"
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue