From 4c485b2af94193ac047a818d38138a9c135ba479 Mon Sep 17 00:00:00 2001 From: luna Date: Sun, 10 May 2026 21:49:10 -0700 Subject: [PATCH] 0.3.0: auto-classifier, manual trigger via .trigger file, GUI tabs (graphs/snapshots/summary), filter, cross-snapshot correlation, NIC delta baseline --- README.md | 36 ++++ packaging/PKGBUILD | 2 +- src/freeze-monitor-gui | 455 ++++++++++++++++++++++++++++++++++------- src/freeze-watcher.sh | 149 +++++++++++++- 4 files changed, 561 insertions(+), 81 deletions(-) diff --git a/README.md b/README.md index da42764..c85e7ed 100644 --- a/README.md +++ b/README.md @@ -116,11 +116,47 @@ The most informative section is **D-state processes** — the `wchan` tells you Cross-reference `wchan` with PSI io and PSI cpu to distinguish lock contention (low PSI) from genuine resource pressure (high PSI). +## Auto-classification + +Each captured snapshot is automatically pattern-matched against known freeze signatures and gets a `## Diagnosis` section at the top: + +- **btrfs lock contention** — `btrfs_lock_root_node` / `btrfs_sync_log` wchans with low PSI io +- **btrfs metadata pressure** — `btrfs_search_slot` without lock contention +- **USB storage stall** — `usb_sg_wait`, `scsi_eh`, `sd_check_events` +- **Memory swap thrash** — `folio_wait_bit_common` with PSI mem +- **Page-fault stall** — `folio_wait_bit_common` without mem pressure (overly aggressive swappiness) +- **CPU oversubscription** — high PSI cpu, few D-state procs +- **NIC IRQ saturation** — recent NET_RX softirq concentrated on one CPU (computed against a rolling 5-minute baseline) +- **Genuine I/O bottleneck** — high PSI io.full + +If no signature matches, the snapshot says "uncertain — review manually". + +## Cross-snapshot summary + +The GUI's **Summary** tab aggregates patterns across all captured snapshots: +- Top diagnoses (which freeze types occur most) +- Top kernel wchans (where processes get stuck) +- Top processes seen in D-state +- Top CPU consumers at capture time +- Captures by hour-of-day distribution + +Useful for spotting "qBittorrent appears in 80% of freezes" type patterns. + +## Live graphs + +The GUI's **Live graphs** tab shows the last hour of load, PSI io/cpu, and D-state count as scrolling line graphs. Red dashed lines mark snapshot capture events so you can correlate spikes with the captured forensics. + +## Manual capture + +Click "📸 Capture Now" in the GUI status bar (or in the tray menu) to trigger a snapshot immediately, regardless of whether thresholds are crossed. Useful when you suspect something is brewing but it hasn't tripped the alarms. + ## Future ideas - Web UI option (no Qt requirement) - Optional perf-profile capture during event - HTML/JSON snapshot output mode +- Snapshot annotations (user notes per capture) +- Anonymized export for sharing ## License diff --git a/packaging/PKGBUILD b/packaging/PKGBUILD index 56efb1e..206f8b6 100644 --- a/packaging/PKGBUILD +++ b/packaging/PKGBUILD @@ -1,6 +1,6 @@ # Maintainer: luna pkgname=freeze-watcher -pkgver=0.2.0 +pkgver=0.3.0 pkgrel=1 pkgdesc="Continuously samples system state and snapshots detailed diagnostics when freeze conditions are detected" arch=('any') diff --git a/src/freeze-monitor-gui b/src/freeze-monitor-gui index f78c7db..9d33a61 100755 --- a/src/freeze-monitor-gui +++ b/src/freeze-monitor-gui @@ -1,25 +1,37 @@ #!/usr/bin/env python3 -"""GUI for freeze-watcher service. Shows live metrics and captured snapshots.""" +"""GUI for freeze-watcher service. + +Layout: + - Status bar (service state + Capture Now) + - Live metrics row (always visible) + - Tabs: Live graphs | Snapshots (filterable list + detail) | Summary +""" -import os import re import subprocess import sys +from collections import Counter, deque from datetime import datetime from pathlib import Path -from PySide6.QtCore import Qt, QTimer -from PySide6.QtGui import QFont, QColor, QAction, QIcon +from PySide6.QtCore import Qt, QTimer, QPointF, Signal +from PySide6.QtGui import QFont, QColor, QAction, QPainter, QPen, QBrush, QPolygonF from PySide6.QtWidgets import ( QApplication, QMainWindow, QWidget, QVBoxLayout, QHBoxLayout, QLabel, QListWidget, QListWidgetItem, QTextEdit, QPushButton, - QSplitter, QGroupBox, QGridLayout, QSystemTrayIcon, QMenu, QStyle + QSplitter, QGroupBox, QGridLayout, QSystemTrayIcon, QMenu, QStyle, + QTabWidget, QLineEdit, QSizePolicy, QMessageBox, ) LOG_DIR = Path("/var/log/freeze-watcher") +TRIGGER = LOG_DIR / ".trigger" SERVICE = "freeze-watcher.service" +HISTORY_SECONDS = 3600 # 1h of 1-second samples for graph +GRAPH_UPDATE_MS = 1000 +# --------------------------------------------------------------------- helpers + def read_psi(name): try: line = (Path("/proc/pressure") / name).read_text().splitlines()[0] @@ -62,6 +74,8 @@ def service_active(): return r.stdout.strip() == "active" +# ----------------------------------------------------------------- widgets + class MetricLabel(QLabel): def __init__(self, name): super().__init__("--") @@ -76,7 +90,8 @@ class MetricLabel(QLabel): def _set_color(self, hex_): self.setStyleSheet( - f"background:#1e1e1e; color:{hex_}; border:1px solid #333; border-radius:6px; padding:6px;" + f"background:#1e1e1e; color:{hex_}; border:1px solid #333; " + f"border-radius:6px; padding:6px;" ) def set_value(self, value, warn, crit, suffix=""): @@ -89,22 +104,168 @@ class MetricLabel(QLabel): self._set_color("#5cdb95") +class LineGraph(QWidget): + """Simple time-series line graph drawn with QPainter. No extra deps.""" + + def __init__(self, label, color="#5cdb95", warn=None, crit=None): + super().__init__() + self.label = label + self.color = QColor(color) + self.warn = warn + self.crit = crit + self.values = deque(maxlen=HISTORY_SECONDS) + self.markers = set() # indices to mark as snapshot events + self.setMinimumHeight(110) + self.setSizePolicy(QSizePolicy.Expanding, QSizePolicy.Preferred) + + def push(self, value, marker=False): + self.values.append(value) + if marker: + self.markers.add(len(self.values) - 1) + # Decay markers as values shift out + new_markers = set() + for m in self.markers: + shifted = m - 1 if len(self.values) == HISTORY_SECONDS else m + if shifted >= 0: + new_markers.add(shifted) + self.markers = new_markers + self.update() + + def paintEvent(self, _): + p = QPainter(self) + p.setRenderHint(QPainter.Antialiasing) + w, h = self.width(), self.height() + + # Background + p.fillRect(0, 0, w, h, QColor("#1a1a1a")) + + if not self.values: + p.setPen(QColor("#666")) + p.drawText(self.rect(), Qt.AlignCenter, f"{self.label} — no data yet") + return + + max_v = max(max(self.values), self.crit or 0, 1) + min_v = 0 + rng = max_v - min_v + + # Title and current value + p.setPen(QColor("#aaa")) + p.setFont(QFont("monospace", 9)) + current = self.values[-1] + p.drawText(8, 16, f"{self.label} current={current:.1f} max={max(self.values):.1f}") + + # Threshold lines + plot_top = 24 + plot_bot = h - 8 + plot_h = plot_bot - plot_top + for thresh, col in [(self.warn, "#665022"), (self.crit, "#5a2222")]: + if thresh is None or thresh > max_v: + continue + y = plot_bot - (thresh - min_v) / rng * plot_h + p.setPen(QPen(QColor(col), 1, Qt.DashLine)) + p.drawLine(0, int(y), w, int(y)) + + # Line + n = len(self.values) + if n < 2: + return + step = w / (n - 1) if n > 1 else 0 + poly = QPolygonF() + for i, v in enumerate(self.values): + x = i * step + y = plot_bot - (v - min_v) / rng * plot_h + poly.append(QPointF(x, y)) + p.setPen(QPen(self.color, 1.5)) + p.drawPolyline(poly) + + # Markers (snapshot captures) + p.setPen(QPen(QColor("#ff5555"), 1, Qt.DotLine)) + for m in self.markers: + x = m * step + p.drawLine(int(x), plot_top, int(x), plot_bot) + + +# ----------------------------------------------------------------- snapshot parsing + +def parse_snapshot(path): + """Extract reason, diagnosis, dstate procs, wchans, top-cpu procs from a snapshot.""" + try: + text = path.read_text(errors="replace") + except Exception: + return None + info = { + "path": str(path), + "mtime": path.stat().st_mtime, + "reason": "", + "diagnoses": [], + "dstate_procs": [], + "wchans": [], + "top_cpu_proc": "", + } + lines = text.splitlines() + i = 0 + while i < len(lines): + ln = lines[i] + if ln.startswith("=== FREEZE EVENT TRIGGERED:"): + info["reason"] = ln.split("TRIGGERED:", 1)[1].strip().rstrip("=").strip() + elif ln.startswith("## Diagnosis"): + i += 1 + while i < len(lines) and lines[i].startswith(" •"): + info["diagnoses"].append(lines[i].lstrip(" •").strip()) + i += 1 + continue + elif ln.startswith("## D-state processes"): + i += 1 + while i < len(lines) and not lines[i].startswith("##"): + parts = lines[i].split() + if len(parts) >= 6 and parts[0] == "D": + info["dstate_procs"].append(parts[4]) # comm + info["wchans"].append(parts[3]) + i += 1 + continue + elif ln.startswith("## Top 15 by CPU"): + j = i + 1 + while j < len(lines) and not lines[j].startswith("##"): + parts = lines[j].split() + if len(parts) >= 6 and parts[1] != "user": + info["top_cpu_proc"] = parts[5] + break + j += 1 + i += 1 + return info + + +# ----------------------------------------------------------------- main window + class FreezeMonitor(QMainWindow): def __init__(self): super().__init__() self.setWindowTitle("Freeze Watcher") - self.resize(1100, 700) + self.resize(1200, 800) central = QWidget() self.setCentralWidget(central) root = QVBoxLayout(central) - # Status bar + # Status row + status_row = QHBoxLayout() self.status_label = QLabel() - self.status_label.setStyleSheet("padding:8px; font-weight:bold;") - root.addWidget(self.status_label) + self.status_label.setStyleSheet("padding:6px; font-weight:bold;") + status_row.addWidget(self.status_label) + status_row.addStretch() + self.btn_capture_now = QPushButton("📸 Capture Now") + self.btn_capture_now.setToolTip("Manually trigger a snapshot capture") + self.btn_capture_now.clicked.connect(self.capture_now) + status_row.addWidget(self.btn_capture_now) + self.btn_start = QPushButton("Start") + self.btn_stop = QPushButton("Stop") + self.btn_start.clicked.connect(lambda: self._svc("start")) + self.btn_stop.clicked.connect(lambda: self._svc("stop")) + status_row.addWidget(self.btn_start) + status_row.addWidget(self.btn_stop) + root.addLayout(status_row) - # Live metrics grid + # Live metrics row metrics_box = QGroupBox("Live metrics (1s refresh)") grid = QGridLayout(metrics_box) labels = ["PSI IO %", "PSI CPU %", "PSI Mem %", "D-state", "Blocked", "Load (1m)"] @@ -116,24 +277,77 @@ class FreezeMonitor(QMainWindow): grid.addWidget(m, 1, i) root.addWidget(metrics_box) - # Buttons - btn_row = QHBoxLayout() - self.btn_start = QPushButton("Start service") - self.btn_stop = QPushButton("Stop service") - self.btn_refresh = QPushButton("Refresh events") - self.btn_open = QPushButton("Open log dir") - self.btn_clear = QPushButton("Delete all snapshots") - for b in (self.btn_start, self.btn_stop, self.btn_refresh, self.btn_open, self.btn_clear): - btn_row.addWidget(b) - btn_row.addStretch() - self.btn_start.clicked.connect(lambda: self._svc("start")) - self.btn_stop.clicked.connect(lambda: self._svc("stop")) - self.btn_refresh.clicked.connect(self.load_events) - self.btn_open.clicked.connect(lambda: subprocess.Popen(["xdg-open", str(LOG_DIR)])) - self.btn_clear.clicked.connect(self.clear_snapshots) - root.addLayout(btn_row) + # Tabs + self.tabs = QTabWidget() + root.addWidget(self.tabs, stretch=1) + + self._build_graphs_tab() + self._build_snapshots_tab() + self._build_summary_tab() + + self._known_events = set() + self._all_snapshots = [] # cached parses + + # Tray + self.tray = QSystemTrayIcon(self.style().standardIcon(QStyle.SP_ComputerIcon), self) + self.tray.setToolTip("Freeze Watcher") + menu = QMenu() + show_act = QAction("Show window", self) + show_act.triggered.connect(self.showNormal) + capture_act = QAction("Capture now", self) + capture_act.triggered.connect(self.capture_now) + quit_act = QAction("Quit", self) + quit_act.triggered.connect(QApplication.quit) + menu.addAction(show_act) + menu.addAction(capture_act) + menu.addSeparator() + menu.addAction(quit_act) + self.tray.setContextMenu(menu) + self.tray.activated.connect(self._tray_clicked) + self.tray.show() + + # Tick timer + self.timer = QTimer() + self.timer.timeout.connect(self.tick) + self.timer.start(GRAPH_UPDATE_MS) + self.load_events() + self.tick() + + # ---- tabs + + def _build_graphs_tab(self): + tab = QWidget() + v = QVBoxLayout(tab) + v.addWidget(QLabel("Last hour — red dashed lines mark snapshot captures")) + self.graph_load = LineGraph("Load (1m)", "#5cdb95", warn=8, crit=16) + self.graph_psi_io = LineGraph("PSI io.some %", "#88ccff", warn=30, crit=60) + self.graph_psi_cpu = LineGraph("PSI cpu.some %", "#ffcc66", warn=30, crit=60) + self.graph_dstate = LineGraph("D-state procs", "#ff8888", warn=3, crit=6) + for g in (self.graph_load, self.graph_psi_io, self.graph_psi_cpu, self.graph_dstate): + v.addWidget(g) + self.tabs.addTab(tab, "Live graphs") + + def _build_snapshots_tab(self): + tab = QWidget() + v = QVBoxLayout(tab) + + filter_row = QHBoxLayout() + filter_row.addWidget(QLabel("Filter:")) + self.filter_edit = QLineEdit() + self.filter_edit.setPlaceholderText("substring match against reason/diagnosis (e.g. btrfs, dstate, usb)") + self.filter_edit.textChanged.connect(self._apply_filter) + filter_row.addWidget(self.filter_edit) + self.btn_refresh = QPushButton("Refresh") + self.btn_refresh.clicked.connect(self.load_events) + self.btn_open = QPushButton("Open log dir") + self.btn_open.clicked.connect(lambda: subprocess.Popen(["xdg-open", str(LOG_DIR)])) + self.btn_clear = QPushButton("Delete all") + self.btn_clear.clicked.connect(self.clear_snapshots) + filter_row.addWidget(self.btn_refresh) + filter_row.addWidget(self.btn_open) + filter_row.addWidget(self.btn_clear) + v.addLayout(filter_row) - # Splitter: list | detail splitter = QSplitter(Qt.Horizontal) self.event_list = QListWidget() self.event_list.itemSelectionChanged.connect(self.show_selected) @@ -142,31 +356,27 @@ class FreezeMonitor(QMainWindow): self.detail.setReadOnly(True) self.detail.setFont(QFont("monospace")) splitter.addWidget(self.detail) - splitter.setSizes([350, 750]) - root.addWidget(splitter, stretch=1) + splitter.setSizes([400, 800]) + v.addWidget(splitter, stretch=1) + self.tabs.addTab(tab, "Snapshots") - self._known_events = set() + def _build_summary_tab(self): + tab = QWidget() + v = QVBoxLayout(tab) + top_row = QHBoxLayout() + top_row.addWidget(QLabel("Cross-snapshot correlation — patterns across all captures")) + top_row.addStretch() + btn = QPushButton("Refresh summary") + btn.clicked.connect(self.refresh_summary) + top_row.addWidget(btn) + v.addLayout(top_row) + self.summary_text = QTextEdit() + self.summary_text.setReadOnly(True) + self.summary_text.setFont(QFont("monospace")) + v.addWidget(self.summary_text) + self.tabs.addTab(tab, "Summary") - # Tray (created before timer so tick() can use it) - self.tray = QSystemTrayIcon(self.style().standardIcon(QStyle.SP_ComputerIcon), self) - self.tray.setToolTip("Freeze Watcher") - menu = QMenu() - show_act = QAction("Show window", self) - show_act.triggered.connect(self.showNormal) - quit_act = QAction("Quit", self) - quit_act.triggered.connect(QApplication.quit) - menu.addAction(show_act) - menu.addAction(quit_act) - self.tray.setContextMenu(menu) - self.tray.activated.connect(self._tray_clicked) - self.tray.show() - - # Timer - self.timer = QTimer() - self.timer.timeout.connect(self.tick) - self.timer.start(1000) - self.load_events() - self.tick() + # ---- actions def _tray_clicked(self, reason): if reason == QSystemTrayIcon.Trigger: @@ -184,6 +394,22 @@ class FreezeMonitor(QMainWindow): subprocess.run(["pkexec", "systemctl", action, SERVICE]) self.tick() + def capture_now(self): + try: + TRIGGER.write_text("manual (gui)") + self.tray.showMessage( + "Freeze Watcher", + "Manual capture requested", + QSystemTrayIcon.Information, + 2000, + ) + except Exception as e: + QMessageBox.warning(self, "Capture failed", + f"Could not write trigger file at {TRIGGER}:\n{e}\n\n" + "Make sure you're in the group owning /var/log/freeze-watcher.") + + # ---- tick + def tick(self): psi_io = read_psi("io") psi_cpu = read_psi("cpu") @@ -202,21 +428,28 @@ class FreezeMonitor(QMainWindow): active = service_active() if active: self.status_label.setText("● Service: ACTIVE") - self.status_label.setStyleSheet("padding:8px; font-weight:bold; color:#5cdb95;") + self.status_label.setStyleSheet("padding:6px; font-weight:bold; color:#5cdb95;") else: self.status_label.setText("● Service: STOPPED") - self.status_label.setStyleSheet("padding:8px; font-weight:bold; color:#ff5555;") + self.status_label.setStyleSheet("padding:6px; font-weight:bold; color:#ff5555;") self.tray.setToolTip( f"Freeze Watcher — {'active' if active else 'stopped'}\n" f"PSI io {int(psi_io)}% / cpu {int(psi_cpu)}% — D:{dstate} blk:{blocked} load:{load:.2f}" ) - # Auto-refresh events list when a new file appears + # Detect new snapshot files for graph markers current = set(p.name for p in LOG_DIR.glob("snap-*.log")) if LOG_DIR.exists() else set() + new = current - self._known_events + is_marker = bool(new) + self.graph_load.push(load, marker=is_marker) + self.graph_psi_io.push(psi_io, marker=is_marker) + self.graph_psi_cpu.push(psi_cpu, marker=is_marker) + self.graph_dstate.push(dstate, marker=is_marker) + if current != self._known_events: - new = current - self._known_events self._known_events = current + self._all_snapshots = [] # invalidate cache self.load_events() for n in new: self.tray.showMessage( @@ -226,24 +459,34 @@ class FreezeMonitor(QMainWindow): 5000, ) + # ---- snapshots tab + def load_events(self): self.event_list.clear() if not LOG_DIR.exists(): return files = sorted(LOG_DIR.glob("snap-*.log"), key=lambda p: p.stat().st_mtime, reverse=True) - for p in files: - reason = "" - try: - first = p.read_text().splitlines()[0] - if "TRIGGERED:" in first: - reason = first.split("TRIGGERED:", 1)[1].strip().rstrip("=").strip() - except Exception: - pass - ts = datetime.fromtimestamp(p.stat().st_mtime).strftime("%Y-%m-%d %H:%M:%S") - item = QListWidgetItem(f"{ts}\n {reason}") - item.setData(Qt.UserRole, str(p)) - self.event_list.addItem(item) + if not self._all_snapshots: + self._all_snapshots = [parse_snapshot(p) for p in files] + self._all_snapshots = [s for s in self._all_snapshots if s] self._known_events = set(p.name for p in files) + self._apply_filter() + + def _apply_filter(self): + needle = self.filter_edit.text().lower().strip() + self.event_list.clear() + for info in self._all_snapshots: + text_blob = (info["reason"] + " " + " ".join(info["diagnoses"])).lower() + if needle and needle not in text_blob: + continue + ts = datetime.fromtimestamp(info["mtime"]).strftime("%Y-%m-%d %H:%M:%S") + diag_short = info["diagnoses"][0][:80] if info["diagnoses"] else "" + label = f"{ts}\n reason: {info['reason']}" + if diag_short: + label += f"\n → {diag_short}" + item = QListWidgetItem(label) + item.setData(Qt.UserRole, info["path"]) + self.event_list.addItem(item) def show_selected(self): items = self.event_list.selectedItems() @@ -258,17 +501,89 @@ class FreezeMonitor(QMainWindow): def clear_snapshots(self): if not LOG_DIR.exists(): return - from PySide6.QtWidgets import QMessageBox if QMessageBox.question(self, "Delete", "Delete all captured snapshots?") != QMessageBox.Yes: return for p in LOG_DIR.glob("snap-*.log"): try: - subprocess.run(["pkexec", "rm", str(p)]) + p.unlink() except Exception: - pass + try: + subprocess.run(["pkexec", "rm", str(p)]) + except Exception: + pass + self._all_snapshots = [] self.load_events() self.detail.clear() + # ---- summary tab + + def refresh_summary(self): + if not self._all_snapshots: + self.load_events() + snaps = self._all_snapshots + if not snaps: + self.summary_text.setPlainText("No snapshots captured yet.") + return + + wchans = Counter() + procs = Counter() + top_cpu = Counter() + diagnoses = Counter() + hours = Counter() + first = None + last = None + for s in snaps: + for w in s["wchans"]: + wchans[w] += 1 + for p in s["dstate_procs"]: + procs[p] += 1 + if s["top_cpu_proc"]: + top_cpu[s["top_cpu_proc"]] += 1 + for d in s["diagnoses"]: + # First clause only, before " — " + key = d.split("—")[0].strip()[:60] + diagnoses[key] += 1 + ts = datetime.fromtimestamp(s["mtime"]) + hours[ts.hour] += 1 + first = min(first, ts) if first else ts + last = max(last, ts) if last else ts + + def topn(ctr, n=10): + total = sum(ctr.values()) or 1 + lines = [] + for k, v in ctr.most_common(n): + pct = 100 * v / total + bar = "█" * int(pct / 2) + lines.append(f" {v:5d} ({pct:5.1f}%) {bar:<25} {k}") + return "\n".join(lines) if lines else " (none)" + + hour_dist = [] + for h in range(24): + count = hours.get(h, 0) + bar = "▇" * count + hour_dist.append(f" {h:02d}:00 {bar} {count}") + + out = [] + out.append(f"Snapshots analyzed: {len(snaps)}") + if first and last: + out.append(f"Time range: {first} → {last}") + out.append("") + out.append("─── Top diagnoses ───────────────────────────────────────") + out.append(topn(diagnoses)) + out.append("") + out.append("─── Top D-state wchans (where stuck) ────────────────────") + out.append(topn(wchans)) + out.append("") + out.append("─── Top processes seen in D-state ───────────────────────") + out.append(topn(procs)) + out.append("") + out.append("─── Top CPU consumers at capture time ───────────────────") + out.append(topn(top_cpu)) + out.append("") + out.append("─── Captures by hour of day ─────────────────────────────") + out.append("\n".join(hour_dist)) + self.summary_text.setPlainText("\n".join(out)) + def main(): app = QApplication(sys.argv) diff --git a/src/freeze-watcher.sh b/src/freeze-watcher.sh index 59d31f4..2410e3c 100755 --- a/src/freeze-watcher.sh +++ b/src/freeze-watcher.sh @@ -1,11 +1,12 @@ #!/bin/bash # Continuously samples system state. Writes a detailed snapshot when -# anomaly thresholds are crossed (D-states, PSI pressure, blocked procs). +# anomaly thresholds are crossed, or when the GUI requests a manual capture. set -u LOG_DIR=/var/log/freeze-watcher CONFIG=/etc/freeze-watcher.conf +TRIGGER_FILE="$LOG_DIR/.trigger" # --- defaults (overridden by /etc/freeze-watcher.conf) ------------------- PSI_IO_THRESHOLD=40 @@ -30,8 +31,6 @@ mkdir -p "$LOG_DIR" last_capture=0 captures_since_prune=0 -# Send a desktop notification to each configured user. -# Daemon runs as root; we sudo into the user's session for DBus access. notify_users() { [ -z "$NOTIFY_USERS" ] && return 0 local body="$1" @@ -48,7 +47,6 @@ notify_users() { done } -# Delete old snapshots based on MAX_SNAPSHOTS and MAX_SNAPSHOT_AGE_DAYS. prune_snapshots() { if [ "${MAX_SNAPSHOT_AGE_DAYS:-0}" -gt 0 ]; then find "$LOG_DIR" -maxdepth 1 -name 'snap-*.log' -type f \ @@ -110,13 +108,108 @@ capture_bpftrace() { ' 2>/dev/null | tail -40 } +# Reads a snapshot file and emits one or more diagnosis lines. +# Pattern-matches D-state wchans and PSI metrics against known freeze types. +classify_snapshot() { + local file="$1" + local diagnoses=() + + local dstate_section + dstate_section=$(awk '/^## D-state processes/{flag=1; next} /^## /{flag=0} flag' "$file") + local softirq + softirq=$(awk '/^## softirq/{flag=1; next} /^## /{flag=0} flag' "$file") + + local psi_io_some psi_io_full psi_cpu_some psi_mem_some + psi_io_some=$(awk '/^\[io\]/{f=1; next} f && /^some/{match($0,/avg10=[0-9.]+/); v=substr($0,RSTART+6,RLENGTH-6); split(v,a,"."); print a[1]+0; exit}' "$file") + psi_io_full=$(awk '/^\[io\]/{f=1; next} f && /^full/{match($0,/avg10=[0-9.]+/); v=substr($0,RSTART+6,RLENGTH-6); split(v,a,"."); print a[1]+0; exit}' "$file") + psi_cpu_some=$(awk '/^\[cpu\]/{f=1; next} f && /^some/{match($0,/avg10=[0-9.]+/); v=substr($0,RSTART+6,RLENGTH-6); split(v,a,"."); print a[1]+0; exit}' "$file") + psi_mem_some=$(awk '/^\[memory\]/{f=1; next} f && /^some/{match($0,/avg10=[0-9.]+/); v=substr($0,RSTART+6,RLENGTH-6); split(v,a,"."); print a[1]+0; exit}' "$file") + + local dcount + dcount=$(echo "$dstate_section" | grep -c "^D ") + + # btrfs lock contention + if echo "$dstate_section" | grep -qE "btrfs_lock_root_node|btrfs_sync_log"; then + if [ "${psi_io_some:-0}" -lt 10 ]; then + diagnoses+=("btrfs metadata lock contention — wchans show btrfs_lock_root_node/sync_log with low PSI io. Typical of qgroups + many snapshots, or heavy fsync churn.") + else + diagnoses+=("btrfs lock contention with concurrent disk I/O") + fi + fi + + # btrfs metadata pressure (less severe) + if echo "$dstate_section" | grep -q "btrfs_search_slot" && \ + ! echo "$dstate_section" | grep -q "btrfs_lock_root_node"; then + diagnoses+=("btrfs metadata pressure — cleaner walking trees, fsyncs queueing") + fi + + # USB stall + if echo "$dstate_section" | grep -qE "usb_sg_wait|scsi_eh|sd_check_events|usb_hcd"; then + diagnoses+=("USB storage stall — one external drive responding slowly") + fi + + # Memory swap thrash + if echo "$dstate_section" | grep -q "folio_wait_bit_common"; then + if [ "${psi_mem_some:-0}" -gt 5 ]; then + diagnoses+=("memory pressure / swap thrash — folio_wait + PSI mem ${psi_mem_some}%") + else + diagnoses+=("page-fault stall on swapped-out pages — apps faulting back from zram/swap (consider lower swappiness)") + fi + fi + + # CPU oversubscription + if [ "${psi_cpu_some:-0}" -gt 30 ] && [ "$dcount" -lt 3 ]; then + diagnoses+=("CPU oversubscription — PSI cpu ${psi_cpu_some}%, few D-state procs. Heavy compile, transcoding, or runaway process competing for cores.") + fi + + # NIC IRQ saturation — use delta against the running baseline so old + # cumulative skew doesn't trigger forever. + if [ -r "$LOG_DIR/.softirq-baseline" ] && [ -n "$(echo "$softirq" | grep '^ *NET_RX:')" ]; then + local cur prev + cur=$(echo "$softirq" | grep '^ *NET_RX:' | sed 's/^ *NET_RX://') + prev=$(awk '/^ *NET_RX:/{sub(/^ *NET_RX:/, ""); print; exit}' "$LOG_DIR/.softirq-baseline") + if [ -n "$prev" ]; then + local max=0 total=0 i=1 + for v in $cur; do + [[ "$v" =~ ^[0-9]+$ ]] || { i=$((i+1)); continue; } + local pv + pv=$(echo "$prev" | awk -v n=$i '{print $n}') + [[ "$pv" =~ ^[0-9]+$ ]] || pv=0 + local d=$((v - pv)) + [ "$d" -lt 0 ] && d=0 + [ "$d" -gt "$max" ] && max=$d + total=$((total + d)) + i=$((i+1)) + done + if [ "$total" -gt 100000 ]; then + local pct=$((max * 100 / total)) + if [ "$pct" -gt 70 ]; then + diagnoses+=("NIC softirq concentrated on one CPU (${pct}% of recent NET_RX) — enable RPS, install irqbalance") + fi + fi + fi + fi + + # Genuine I/O bottleneck + if [ "${psi_io_full:-0}" -gt 30 ]; then + diagnoses+=("genuine I/O bottleneck — PSI io full ${psi_io_full}%, device truly can't keep up") + fi + + if [ ${#diagnoses[@]} -eq 0 ]; then + echo " • uncertain — no known signature matched; review wchans and metrics manually" + else + for d in "${diagnoses[@]}"; do + echo " • $d" + done + fi +} + capture_snapshot() { local reason="$1" - local file="$LOG_DIR/snap-$(date +%Y%m%d-%H%M%S).log" + local body="$LOG_DIR/.snap-body-$$.tmp" + local final="$LOG_DIR/snap-$(date +%Y%m%d-%H%M%S).log" + { - echo "=== FREEZE EVENT TRIGGERED: $reason ===" - echo "Time: $(date -Iseconds)" - echo echo "## /proc/loadavg" cat /proc/loadavg echo @@ -166,17 +259,47 @@ capture_snapshot() { echo echo "## journal warnings/errors last 60s" journalctl --since "60 seconds ago" -p warning --no-pager 2>&1 | tail -40 - } > "$file" 2>&1 - echo "$(date -Iseconds) Captured snapshot to $file (reason: $reason)" >> "$LOG_DIR/events.log" + } > "$body" 2>&1 + + # Build final file: header + diagnosis (computed from body) + body + { + echo "=== FREEZE EVENT TRIGGERED: $reason ===" + echo "Time: $(date -Iseconds)" + echo + echo "## Diagnosis (auto-classified)" + classify_snapshot "$body" + echo + cat "$body" + } > "$final" + + rm -f "$body" + echo "$(date -Iseconds) Captured snapshot to $final (reason: $reason)" >> "$LOG_DIR/events.log" notify_users "$reason" } echo "$(date -Iseconds) freeze-watcher started (pid $$)" >> "$LOG_DIR/events.log" prune_snapshots +# Initial softirq baseline so classifier can compute deltas +cp /proc/softirqs "$LOG_DIR/.softirq-baseline" 2>/dev/null +softirq_baseline_at=$(date +%s) + while true; do now=$(date +%s) + # Manual trigger from GUI (file-based, no auth required for users in log group) + if [ -f "$TRIGGER_FILE" ]; then + manual_reason="manual" + if [ -s "$TRIGGER_FILE" ]; then + manual_reason=$(head -c 100 "$TRIGGER_FILE" | tr -d '\n') + fi + rm -f "$TRIGGER_FILE" + capture_snapshot "$manual_reason" & + last_capture=$now + sleep "$SAMPLE_INTERVAL" + continue + fi + psi_io=$(awk '/^some/{for(i=1;i<=NF;i++) if($i~/^avg10=/){gsub(/avg10=/,"",$i); print int($i+0.5); exit}}' /proc/pressure/io 2>/dev/null) psi_cpu=$(awk '/^some/{for(i=1;i<=NF;i++) if($i~/^avg10=/){gsub(/avg10=/,"",$i); print int($i+0.5); exit}}' /proc/pressure/cpu 2>/dev/null) dstate=$(ps -eo state --no-headers | grep -c "^D") @@ -200,5 +323,11 @@ while true; do fi fi + # Refresh softirq baseline every 5 minutes so classifier deltas reflect recent activity + if [ $((now - softirq_baseline_at)) -ge 300 ]; then + cp /proc/softirqs "$LOG_DIR/.softirq-baseline" 2>/dev/null + softirq_baseline_at=$now + fi + sleep "$SAMPLE_INTERVAL" done