Initial commit: freeze-watcher daemon, GUI, systemd unit, PKGBUILD

src/freeze-watcher.sh

@@ -0,0 +1,99 @@
+#!/bin/bash
+# Continuously samples system state. Writes a detailed snapshot when
+# anomaly thresholds are crossed (D-states, PSI pressure, blocked procs).
+
+LOG_DIR=/var/log/freeze-watcher
+mkdir -p "$LOG_DIR"
+
+# Thresholds
+PSI_IO_THRESHOLD=40       # io pressure avg10 % to trigger
+PSI_CPU_THRESHOLD=50      # cpu pressure avg10 % to trigger
+DSTATE_THRESHOLD=5        # processes in D state
+BLOCKED_THRESHOLD=5       # procs_blocked from /proc/stat
+LOAD_THRESHOLD=20         # 1-min load average
+COOLDOWN=30               # seconds between captures
+SAMPLE_INTERVAL=1         # seconds between samples
+
+last_capture=0
+
+capture_snapshot() {
+    local reason="$1"
+    local now=$(date +%s)
+    local file="$LOG_DIR/snap-$(date +%Y%m%d-%H%M%S).log"
+    {
+        echo "=== FREEZE EVENT TRIGGERED: $reason ==="
+        echo "Time: $(date -Iseconds)"
+        echo
+        echo "## /proc/loadavg"
+        cat /proc/loadavg
+        echo
+        echo "## /proc/stat (procs_running, procs_blocked)"
+        grep -E "procs_running|procs_blocked|intr|ctxt" /proc/stat | head -5
+        echo
+        echo "## PSI cpu/io/memory"
+        for k in cpu io memory; do
+            echo "[$k]"; cat /proc/pressure/$k 2>/dev/null
+        done
+        echo
+        echo "## D-state processes (with kernel wchan = where they're stuck)"
+        ps -eo state,pid,user,wchan:25,comm,args --no-headers | awk '$1=="D"' | head -30
+        echo
+        echo "## Top 15 by CPU"
+        ps -eo pid,user,pcpu,pmem,state,comm --no-headers --sort=-pcpu | head -15
+        echo
+        echo "## Top 15 by RSS memory"
+        ps -eo pid,user,pcpu,pmem,rss,comm --no-headers --sort=-rss | head -15
+        echo
+        echo "## /proc/meminfo summary"
+        grep -E "MemTotal|MemFree|MemAvailable|Dirty|Writeback|Slab|Cached|Buffers" /proc/meminfo
+        echo
+        echo "## iostat (1s snapshot)"
+        iostat -xt 1 2 2>&1 | tail -50
+        echo
+        echo "## interrupts (top by total)"
+        awk 'NR==1{next} {sum=0; for(i=2;i<=NF-2;i++) sum+=$i; if(sum>1000) print sum, $0}' /proc/interrupts | sort -rn | head -15
+        echo
+        echo "## softirq"
+        cat /proc/softirqs
+        echo
+        echo "## Recent dmesg (last 50 lines)"
+        dmesg -T | tail -50
+        echo
+        echo "## journal warnings/errors last 60s"
+        journalctl --since "60 seconds ago" -p warning --no-pager 2>&1 | tail -40
+    } > "$file" 2>&1
+    echo "$(date -Iseconds) Captured snapshot to $file (reason: $reason)" >> "$LOG_DIR/events.log"
+}
+
+echo "$(date -Iseconds) freeze-watcher started (pid $$)" >> "$LOG_DIR/events.log"
+
+while true; do
+    now=$(date +%s)
+
+    # PSI pressure - parse "some avg10=X.XX"
+    psi_io=$(awk '/^some/{for(i=1;i<=NF;i++) if($i~/^avg10=/){gsub(/avg10=/,"",$i); print int($i+0.5); exit}}' /proc/pressure/io 2>/dev/null)
+    psi_cpu=$(awk '/^some/{for(i=1;i<=NF;i++) if($i~/^avg10=/){gsub(/avg10=/,"",$i); print int($i+0.5); exit}}' /proc/pressure/cpu 2>/dev/null)
+
+    # D-state count
+    dstate=$(ps -eo state --no-headers | grep -c "^D")
+
+    # /proc/stat blocked
+    blocked=$(awk '/^procs_blocked/{print $2}' /proc/stat)
+
+    # Load average
+    load1=$(awk '{print int($1+0.5)}' /proc/loadavg)
+
+    triggered=""
+    [ "${psi_io:-0}" -ge "$PSI_IO_THRESHOLD" ] && triggered="psi_io=$psi_io%"
+    [ "${psi_cpu:-0}" -ge "$PSI_CPU_THRESHOLD" ] && triggered="${triggered:+$triggered, }psi_cpu=$psi_cpu%"
+    [ "${dstate:-0}" -ge "$DSTATE_THRESHOLD" ] && triggered="${triggered:+$triggered, }dstate=$dstate"
+    [ "${blocked:-0}" -ge "$BLOCKED_THRESHOLD" ] && triggered="${triggered:+$triggered, }blocked=$blocked"
+    [ "${load1:-0}" -ge "$LOAD_THRESHOLD" ] && triggered="${triggered:+$triggered, }load1=$load1"
+
+    if [ -n "$triggered" ] && [ $((now - last_capture)) -ge "$COOLDOWN" ]; then
+        capture_snapshot "$triggered" &
+        last_capture=$now
+    fi
+
+    sleep "$SAMPLE_INTERVAL"
+done