0.2.0: config file, auto-prune, notify-send, cgroup PSI, kernel stacks, bpftrace, tray tooltip

Makefile

@@ -2,6 +2,7 @@ PREFIX        ?= /usr/local
 BINDIR        := $(PREFIX)/bin
 APPDIR        := $(PREFIX)/share/applications
 SYSTEMDDIR    := /etc/systemd/system
+CONFDIR       := /etc
 LOGDIR        := /var/log/freeze-watcher
 LOG_GROUP     ?= $(shell id -gn)
 
@@ -12,6 +13,13 @@ install:
 	install -Dm755 src/freeze-monitor-gui   $(DESTDIR)$(BINDIR)/freeze-monitor-gui
 	install -Dm644 systemd/freeze-watcher.service $(DESTDIR)$(SYSTEMDDIR)/freeze-watcher.service
 	install -Dm644 desktop/freeze-monitor.desktop $(DESTDIR)$(APPDIR)/freeze-monitor.desktop
+	@if [ ! -e "$(DESTDIR)$(CONFDIR)/freeze-watcher.conf" ]; then \
+		install -Dm644 config/freeze-watcher.conf $(DESTDIR)$(CONFDIR)/freeze-watcher.conf; \
+		echo "Installed default config at $(CONFDIR)/freeze-watcher.conf"; \
+	else \
+		install -Dm644 config/freeze-watcher.conf $(DESTDIR)$(CONFDIR)/freeze-watcher.conf.new; \
+		echo "Existing config preserved; new template at $(CONFDIR)/freeze-watcher.conf.new"; \
+	fi
 	install -dm775 -g $(LOG_GROUP) $(DESTDIR)$(LOGDIR)
 	@echo "Installed. Run 'make enable' to start the service."
 
@@ -20,7 +28,8 @@ uninstall:
 	rm -f $(DESTDIR)$(BINDIR)/freeze-monitor-gui
 	rm -f $(DESTDIR)$(SYSTEMDDIR)/freeze-watcher.service
 	rm -f $(DESTDIR)$(APPDIR)/freeze-monitor.desktop
-	@echo "Uninstalled. Logs at $(LOGDIR) preserved."
+	rm -f $(DESTDIR)$(CONFDIR)/freeze-watcher.conf.new
+	@echo "Uninstalled. Config at $(CONFDIR)/freeze-watcher.conf and logs at $(LOGDIR) preserved."
 
 enable:
 	systemctl daemon-reload

README.md

@@ -64,20 +64,33 @@ sudo systemctl restart freeze-watcher.service
 
 If you're using the Arch package: `./packaging/build-package.sh && sudo pacman -U *.pkg.tar.zst` will rebuild and upgrade in place.
 
-## Tuning thresholds
+## Configuration
 
-Open `src/freeze-watcher.sh` and edit the constants at the top:
+Edit `/etc/freeze-watcher.conf`. After changes:
 
 ```
-PSI_IO_THRESHOLD=40       # io pressure avg10 % to trigger
+sudo systemctl restart freeze-watcher.service
-PSI_CPU_THRESHOLD=50      # cpu pressure avg10 % to trigger
-DSTATE_THRESHOLD=5        # processes in D state
-BLOCKED_THRESHOLD=5       # procs_blocked from /proc/stat
-LOAD_THRESHOLD=20         # 1-min load average
-COOLDOWN=30               # seconds between captures
-SAMPLE_INTERVAL=1         # seconds between samples
 ```
 
+Options:
+
+| Variable | Default | Purpose |
+|---|---|---|
+| `PSI_IO_THRESHOLD` | 40 | io pressure avg10 % to trigger |
+| `PSI_CPU_THRESHOLD` | 50 | cpu pressure avg10 % to trigger |
+| `DSTATE_THRESHOLD` | 5 | processes in uninterruptible sleep |
+| `BLOCKED_THRESHOLD` | 5 | procs_blocked from /proc/stat |
+| `LOAD_THRESHOLD` | 20 | 1-min load average |
+| `SAMPLE_INTERVAL` | 1 | seconds between samples |
+| `COOLDOWN` | 30 | min seconds between captures |
+| `MAX_SNAPSHOTS` | 200 | auto-delete oldest beyond this count (0 disables) |
+| `MAX_SNAPSHOT_AGE_DAYS` | 30 | auto-delete older than this (0 disables) |
+| `NOTIFY_USERS` | "" | comma-separated users to notify-send on capture |
+| `NOTIFY_URGENCY` | normal | low / normal / critical |
+| `CAPTURE_KERNEL_STACKS` | 1 | dump /proc/[pid]/stack for D-state procs |
+| `CAPTURE_BPFTRACE` | 0 | run bpftrace one-shot (requires bpftrace) |
+| `CAPTURE_CGROUP_PSI` | 1 | walk cgroups for top per-cgroup PSI |
+
 ## GUI
 
 Launch from app menu (search "Freeze Watcher") or `freeze-monitor-gui`.
@@ -105,13 +118,9 @@ Cross-reference `wchan` with PSI io and PSI cpu to distinguish lock contention (
 
 ## Future ideas
 
-- Per-cgroup PSI capture
-- eBPF tracing for long-running syscalls
 - Web UI option (no Qt requirement)
-- Auto-prune old snapshots
-- Configurable `/etc/freeze-watcher.conf` instead of inline thresholds
-- Integration with desktop notifications via `notify-send`
 - Optional perf-profile capture during event
+- HTML/JSON snapshot output mode
 
 ## License
 

config/freeze-watcher.conf

@@ -0,0 +1,31 @@
+# freeze-watcher configuration
+# Sourced by /usr/local/bin/freeze-watcher.sh at startup.
+# Lines are bash variable assignments. Restart the service after editing:
+#   sudo systemctl restart freeze-watcher.service
+
+# --- thresholds ----------------------------------------------------------
+# A snapshot is captured when ANY of these is exceeded.
+PSI_IO_THRESHOLD=40        # io pressure avg10 % to trigger
+PSI_CPU_THRESHOLD=50       # cpu pressure avg10 % to trigger
+DSTATE_THRESHOLD=5         # processes in uninterruptible sleep
+BLOCKED_THRESHOLD=5        # procs_blocked from /proc/stat
+LOAD_THRESHOLD=20          # 1-min load average
+
+# --- timing --------------------------------------------------------------
+SAMPLE_INTERVAL=1          # seconds between samples
+COOLDOWN=30                # min seconds between captures
+
+# --- retention -----------------------------------------------------------
+MAX_SNAPSHOTS=200          # auto-delete oldest beyond this count (0 = no cap)
+MAX_SNAPSHOT_AGE_DAYS=30   # auto-delete older than this (0 = no age cap)
+
+# --- notifications -------------------------------------------------------
+# Send a desktop notification when a snapshot is captured.
+# Comma-separated list of users to notify (must be logged into a graphical session).
+NOTIFY_USERS=""            # e.g. "luna" or "luna,alice"
+NOTIFY_URGENCY=normal      # low, normal, critical
+
+# --- capture options -----------------------------------------------------
+CAPTURE_KERNEL_STACKS=1    # dump /proc/[pid]/stack for D-state procs (free, very useful)
+CAPTURE_BPFTRACE=0         # run bpftrace one-shot during capture (requires bpftrace)
+CAPTURE_CGROUP_PSI=1       # walk /sys/fs/cgroup for top PSI cgroups

packaging/PKGBUILD

@@ -1,16 +1,22 @@
 # Maintainer: luna <anassaeneroi@pm.me>
 pkgname=freeze-watcher
-pkgver=0.1.0
+pkgver=0.2.0
 pkgrel=1
 pkgdesc="Continuously samples system state and snapshots detailed diagnostics when freeze conditions are detected"
 arch=('any')
 url="https://github.com/luna/freeze-watcher"
 license=('MIT')
-depends=('bash' 'systemd' 'python' 'pyside6' 'sysstat' 'procps-ng' 'util-linux')
+depends=('bash' 'systemd' 'python' 'pyside6' 'sysstat' 'procps-ng' 'util-linux' 'libnotify' 'sudo')
-optdepends=('xdg-utils: open log directory from GUI')
+optdepends=(
+    'xdg-utils: open log directory from GUI'
+    'bpftrace: kernel stack sampling during capture (CAPTURE_BPFTRACE=1)'
+)
 source=("$pkgname-$pkgver.tar.gz::file://$PWD/../$pkgname-$pkgver.tar.gz")
 sha256sums=('SKIP')
-backup=('etc/systemd/system/freeze-watcher.service')
+backup=(
+    'etc/systemd/system/freeze-watcher.service'
+    'etc/freeze-watcher.conf'
+)
 
 package() {
     cd "$srcdir/$pkgname-$pkgver"

packaging/build-package.sh

@@ -14,7 +14,7 @@ trap "rm -rf $tmp" EXIT
 
 # Stage source as a tarball (PKGBUILD expects it as $pkgname-$pkgver.tar.gz)
 mkdir "$tmp/$PKGNAME-$PKGVER"
-cp -r src systemd desktop Makefile README.md LICENSE "$tmp/$PKGNAME-$PKGVER/" 2>/dev/null || true
+cp -r src systemd desktop config Makefile README.md LICENSE "$tmp/$PKGNAME-$PKGVER/" 2>/dev/null || true
 tar czf "$tmp/$PKGNAME-$PKGVER.tar.gz" -C "$tmp" "$PKGNAME-$PKGVER"
 
 cp packaging/PKGBUILD "$tmp/PKGBUILD"

src/freeze-monitor-gui

@@ -149,6 +149,7 @@ class FreezeMonitor(QMainWindow):
 
         # Tray (created before timer so tick() can use it)
         self.tray = QSystemTrayIcon(self.style().standardIcon(QStyle.SP_ComputerIcon), self)
+        self.tray.setToolTip("Freeze Watcher")
         menu = QMenu()
         show_act = QAction("Show window", self)
         show_act.triggered.connect(self.showNormal)
@@ -198,13 +199,19 @@ class FreezeMonitor(QMainWindow):
         self.metrics["Blocked"].set_value(blocked, 3, 6, "")
         self.metrics["Load (1m)"].set_value(int(load), 8, 16, "")
 
-        if service_active():
+        active = service_active()
+        if active:
             self.status_label.setText("● Service: ACTIVE")
             self.status_label.setStyleSheet("padding:8px; font-weight:bold; color:#5cdb95;")
         else:
             self.status_label.setText("● Service: STOPPED")
             self.status_label.setStyleSheet("padding:8px; font-weight:bold; color:#ff5555;")
 
+        self.tray.setToolTip(
+            f"Freeze Watcher — {'active' if active else 'stopped'}\n"
+            f"PSI io {int(psi_io)}% / cpu {int(psi_cpu)}% — D:{dstate} blk:{blocked} load:{load:.2f}"
+        )
+
         # Auto-refresh events list when a new file appears
         current = set(p.name for p in LOG_DIR.glob("snap-*.log")) if LOG_DIR.exists() else set()
         if current != self._known_events:

src/freeze-watcher.sh

@@ -2,23 +2,116 @@
 # Continuously samples system state. Writes a detailed snapshot when
 # anomaly thresholds are crossed (D-states, PSI pressure, blocked procs).
 
+set -u
+
 LOG_DIR=/var/log/freeze-watcher
+CONFIG=/etc/freeze-watcher.conf
+
+# --- defaults (overridden by /etc/freeze-watcher.conf) -------------------
+PSI_IO_THRESHOLD=40
+PSI_CPU_THRESHOLD=50
+DSTATE_THRESHOLD=5
+BLOCKED_THRESHOLD=5
+LOAD_THRESHOLD=20
+COOLDOWN=30
+SAMPLE_INTERVAL=1
+MAX_SNAPSHOTS=200
+MAX_SNAPSHOT_AGE_DAYS=30
+NOTIFY_USERS=""
+NOTIFY_URGENCY=normal
+CAPTURE_KERNEL_STACKS=1
+CAPTURE_BPFTRACE=0
+CAPTURE_CGROUP_PSI=1
+
+[ -f "$CONFIG" ] && . "$CONFIG"
+
 mkdir -p "$LOG_DIR"
 
-# Thresholds
-PSI_IO_THRESHOLD=40       # io pressure avg10 % to trigger
-PSI_CPU_THRESHOLD=50      # cpu pressure avg10 % to trigger
-DSTATE_THRESHOLD=5        # processes in D state
-BLOCKED_THRESHOLD=5       # procs_blocked from /proc/stat
-LOAD_THRESHOLD=20         # 1-min load average
-COOLDOWN=30               # seconds between captures
-SAMPLE_INTERVAL=1         # seconds between samples
-
 last_capture=0
+captures_since_prune=0
+
+# Send a desktop notification to each configured user.
+# Daemon runs as root; we sudo into the user's session for DBus access.
+notify_users() {
+    [ -z "$NOTIFY_USERS" ] && return 0
+    local body="$1"
+    local IFS=,
+    for user in $NOTIFY_USERS; do
+        user=$(echo "$user" | tr -d '[:space:]')
+        [ -z "$user" ] && continue
+        local uid
+        uid=$(id -u "$user" 2>/dev/null) || continue
+        sudo -u "$user" \
+            DBUS_SESSION_BUS_ADDRESS="unix:path=/run/user/$uid/bus" \
+            notify-send -u "$NOTIFY_URGENCY" -a "freeze-watcher" \
+                "Freeze captured" "$body" 2>/dev/null &
+    done
+}
+
+# Delete old snapshots based on MAX_SNAPSHOTS and MAX_SNAPSHOT_AGE_DAYS.
+prune_snapshots() {
+    if [ "${MAX_SNAPSHOT_AGE_DAYS:-0}" -gt 0 ]; then
+        find "$LOG_DIR" -maxdepth 1 -name 'snap-*.log' -type f \
+            -mtime "+$MAX_SNAPSHOT_AGE_DAYS" -delete 2>/dev/null
+    fi
+    if [ "${MAX_SNAPSHOTS:-0}" -gt 0 ]; then
+        local files
+        files=$(ls -1t "$LOG_DIR"/snap-*.log 2>/dev/null | tail -n "+$((MAX_SNAPSHOTS+1))")
+        [ -n "$files" ] && echo "$files" | xargs -r rm -f
+    fi
+}
+
+capture_kernel_stacks() {
+    echo "## Kernel stack traces of D-state procs"
+    ps -eo pid,state --no-headers | awk '$2=="D"{print $1}' | while read -r pid; do
+        local comm
+        comm=$(cat "/proc/$pid/comm" 2>/dev/null) || continue
+        echo "--- pid=$pid comm=$comm ---"
+        cat "/proc/$pid/stack" 2>/dev/null | head -20
+    done
+}
+
+capture_cgroup_psi() {
+    echo "## Top cgroups by PSI io.some avg10"
+    {
+        for f in /sys/fs/cgroup/io.pressure \
+                 /sys/fs/cgroup/*/io.pressure \
+                 /sys/fs/cgroup/*/*/io.pressure; do
+            [ -r "$f" ] || continue
+            local v
+            v=$(awk '/^some/{for(i=1;i<=NF;i++) if($i~/^avg10=/){gsub(/avg10=/,"",$i); print $i; exit}}' "$f" 2>/dev/null)
+            [ -z "$v" ] && continue
+            [ "$v" = "0.00" ] && continue
+            printf "%s\t%s\n" "$v" "$(dirname "$f" | sed 's|/sys/fs/cgroup||')"
+        done
+    } 2>/dev/null | sort -rn | head -10
+    echo
+    echo "## Top cgroups by PSI cpu.some avg10"
+    {
+        for f in /sys/fs/cgroup/cpu.pressure \
+                 /sys/fs/cgroup/*/cpu.pressure \
+                 /sys/fs/cgroup/*/*/cpu.pressure; do
+            [ -r "$f" ] || continue
+            local v
+            v=$(awk '/^some/{for(i=1;i<=NF;i++) if($i~/^avg10=/){gsub(/avg10=/,"",$i); print $i; exit}}' "$f" 2>/dev/null)
+            [ -z "$v" ] && continue
+            [ "$v" = "0.00" ] && continue
+            printf "%s\t%s\n" "$v" "$(dirname "$f" | sed 's|/sys/fs/cgroup||')"
+        done
+    } 2>/dev/null | sort -rn | head -10
+}
+
+capture_bpftrace() {
+    command -v bpftrace >/dev/null 2>&1 || { echo "bpftrace not installed"; return; }
+    echo "## bpftrace: top kernel stacks for io_schedule waits over 2s"
+    timeout 3 bpftrace -e '
+        kprobe:io_schedule { @[kstack(5)] = count(); }
+        interval:s:2 { exit(); }
+    ' 2>/dev/null | tail -40
+}
 
 capture_snapshot() {
     local reason="$1"
-    local now=$(date +%s)
     local file="$LOG_DIR/snap-$(date +%Y%m%d-%H%M%S).log"
     {
         echo "=== FREEZE EVENT TRIGGERED: $reason ==="
@@ -35,9 +128,17 @@ capture_snapshot() {
             echo "[$k]"; cat /proc/pressure/$k 2>/dev/null
         done
         echo
-        echo "## D-state processes (with kernel wchan = where they're stuck)"
+        if [ "${CAPTURE_CGROUP_PSI:-1}" = "1" ]; then
+            capture_cgroup_psi
+            echo
+        fi
+        echo "## D-state processes (with kernel wchan)"
         ps -eo state,pid,user,wchan:25,comm,args --no-headers | awk '$1=="D"' | head -30
         echo
+        if [ "${CAPTURE_KERNEL_STACKS:-1}" = "1" ]; then
+            capture_kernel_stacks
+            echo
+        fi
         echo "## Top 15 by CPU"
         ps -eo pid,user,pcpu,pmem,state,comm --no-headers --sort=-pcpu | head -15
         echo
@@ -56,6 +157,10 @@ capture_snapshot() {
         echo "## softirq"
         cat /proc/softirqs
         echo
+        if [ "${CAPTURE_BPFTRACE:-0}" = "1" ]; then
+            capture_bpftrace
+            echo
+        fi
         echo "## Recent dmesg (last 50 lines)"
         dmesg -T | tail -50
         echo
@@ -63,24 +168,19 @@ capture_snapshot() {
         journalctl --since "60 seconds ago" -p warning --no-pager 2>&1 | tail -40
     } > "$file" 2>&1
     echo "$(date -Iseconds) Captured snapshot to $file (reason: $reason)" >> "$LOG_DIR/events.log"
+    notify_users "$reason"
 }
 
 echo "$(date -Iseconds) freeze-watcher started (pid $$)" >> "$LOG_DIR/events.log"
+prune_snapshots
 
 while true; do
     now=$(date +%s)
 
-    # PSI pressure - parse "some avg10=X.XX"
     psi_io=$(awk '/^some/{for(i=1;i<=NF;i++) if($i~/^avg10=/){gsub(/avg10=/,"",$i); print int($i+0.5); exit}}' /proc/pressure/io 2>/dev/null)
     psi_cpu=$(awk '/^some/{for(i=1;i<=NF;i++) if($i~/^avg10=/){gsub(/avg10=/,"",$i); print int($i+0.5); exit}}' /proc/pressure/cpu 2>/dev/null)
-
-    # D-state count
     dstate=$(ps -eo state --no-headers | grep -c "^D")
-
-    # /proc/stat blocked
     blocked=$(awk '/^procs_blocked/{print $2}' /proc/stat)
-
-    # Load average
     load1=$(awk '{print int($1+0.5)}' /proc/loadavg)
 
     triggered=""
@@ -93,6 +193,11 @@ while true; do
     if [ -n "$triggered" ] && [ $((now - last_capture)) -ge "$COOLDOWN" ]; then
         capture_snapshot "$triggered" &
         last_capture=$now
+        captures_since_prune=$((captures_since_prune+1))
+        if [ "$captures_since_prune" -ge 20 ]; then
+            prune_snapshots &
+            captures_since_prune=0
+        fi
     fi
 
     sleep "$SAMPLE_INTERVAL"