diff --git a/packaging/PKGBUILD b/packaging/PKGBUILD index 6d15351..746a398 100644 --- a/packaging/PKGBUILD +++ b/packaging/PKGBUILD @@ -1,6 +1,6 @@ # Maintainer: luna pkgname=freeze-watcher -pkgver=0.4.0 +pkgver=0.4.1 pkgrel=1 pkgdesc="Continuously samples system state and snapshots detailed diagnostics when freeze conditions are detected" arch=('any') diff --git a/src/freeze-watcher.sh b/src/freeze-watcher.sh index dc52a95..ca1fa6e 100755 --- a/src/freeze-watcher.sh +++ b/src/freeze-watcher.sh @@ -166,6 +166,8 @@ classify_snapshot() { # NIC IRQ saturation — use delta against the running baseline so old # cumulative skew doesn't trigger forever. + # With RPS enabled, per-flow concentration on one CPU is normal up to ~85%. + # Only flag when no RPS spreading is happening at all, or it's truly extreme. if [ -r "$LOG_DIR/.softirq-baseline" ] && [ -n "$(echo "$softirq" | grep '^ *NET_RX:')" ]; then local cur prev cur=$(echo "$softirq" | grep '^ *NET_RX:' | sed 's/^ *NET_RX://') @@ -185,8 +187,20 @@ classify_snapshot() { done if [ "$total" -gt 100000 ]; then local pct=$((max * 100 / total)) - if [ "$pct" -gt 70 ]; then - diagnoses+=("NIC softirq concentrated on one CPU (${pct}% of recent NET_RX) — enable RPS, install irqbalance") + # Check if RPS is configured (any non-zero, non-single-bit value) + local rps_active=0 + for rps in /sys/class/net/*/queues/rx-0/rps_cpus; do + [ -r "$rps" ] || continue + local v + v=$(cat "$rps" 2>/dev/null | tr -d ',') + # Strip leading zeros, count hex bits set + [ "$v" != "0" ] && [ "$v" != "00" ] && [ "$v" != "000" ] \ + && [ "$v" != "0000" ] && rps_active=1 + done + if [ "$pct" -gt 90 ] && [ "$rps_active" = "0" ]; then + diagnoses+=("NIC softirq saturating one CPU (${pct}% of recent NET_RX, RPS disabled) — enable RPS, install irqbalance") + elif [ "$pct" -gt 95 ]; then + diagnoses+=("NIC softirq extreme concentration (${pct}% of recent NET_RX) — even with RPS, single-flow saturation; consider enabling RFS or reducing peer connections") fi fi fi @@ -211,6 +225,17 @@ capture_snapshot() { local body="$LOG_DIR/.snap-body-$$.tmp" local final="$LOG_DIR/snap-$(date +%Y%m%d-%H%M%S).log" + # CAPTURE D-STATE FIRST - blocked spikes are often <1s long, so we need + # to grab process state before reading slower /proc/* files + local dstate_immediate + dstate_immediate=$(ps -eo state,pid,user,wchan:25,comm,args --no-headers 2>/dev/null | awk '$1=="D"' | head -30) + local dstate_t1 + sleep 0.05 + dstate_t1=$(ps -eo state,pid,user,wchan:25,comm,args --no-headers 2>/dev/null | awk '$1=="D"' | head -30) + local dstate_t2 + sleep 0.05 + dstate_t2=$(ps -eo state,pid,user,wchan:25,comm,args --no-headers 2>/dev/null | awk '$1=="D"' | head -30) + { echo "## /proc/loadavg" cat /proc/loadavg @@ -228,7 +253,11 @@ capture_snapshot() { echo fi echo "## D-state processes (with kernel wchan)" - ps -eo state,pid,user,wchan:25,comm,args --no-headers | awk '$1=="D"' | head -30 + # Use the snapshot taken before /proc reads to catch transient blockers. + # Combine 3 rapid samples (~100ms apart) and dedupe by pid. + echo "$dstate_immediate" + echo "$dstate_t1" | grep -vF "$dstate_immediate" 2>/dev/null + echo "$dstate_t2" | grep -vF "$dstate_immediate" | grep -vF "$dstate_t1" 2>/dev/null echo if [ "${CAPTURE_KERNEL_STACKS:-1}" = "1" ]; then capture_kernel_stacks