0.3.0: auto-classifier, manual trigger via .trigger file, GUI tabs (graphs/snapshots/summary), filter, cross-snapshot correlation, NIC delta baseline
This commit is contained in:
luna
parent
41dcb0f577
commit
4c485b2af9
4 changed files with 561 additions and 81 deletions
36
README.md
36
README.md
|
|
@ -116,11 +116,47 @@ The most informative section is **D-state processes** — the `wchan` tells you
|
|||
|
||||
Cross-reference `wchan` with PSI io and PSI cpu to distinguish lock contention (low PSI) from genuine resource pressure (high PSI).
|
||||
|
||||
## Auto-classification
|
||||
|
||||
Each captured snapshot is automatically pattern-matched against known freeze signatures and gets a `## Diagnosis` section at the top:
|
||||
|
||||
- **btrfs lock contention** — `btrfs_lock_root_node` / `btrfs_sync_log` wchans with low PSI io
|
||||
- **btrfs metadata pressure** — `btrfs_search_slot` without lock contention
|
||||
- **USB storage stall** — `usb_sg_wait`, `scsi_eh`, `sd_check_events`
|
||||
- **Memory swap thrash** — `folio_wait_bit_common` with PSI mem
|
||||
- **Page-fault stall** — `folio_wait_bit_common` without mem pressure (overly aggressive swappiness)
|
||||
- **CPU oversubscription** — high PSI cpu, few D-state procs
|
||||
- **NIC IRQ saturation** — recent NET_RX softirq concentrated on one CPU (computed against a rolling 5-minute baseline)
|
||||
- **Genuine I/O bottleneck** — high PSI io.full
|
||||
|
||||
If no signature matches, the snapshot says "uncertain — review manually".
|
||||
|
||||
## Cross-snapshot summary
|
||||
|
||||
The GUI's **Summary** tab aggregates patterns across all captured snapshots:
|
||||
- Top diagnoses (which freeze types occur most)
|
||||
- Top kernel wchans (where processes get stuck)
|
||||
- Top processes seen in D-state
|
||||
- Top CPU consumers at capture time
|
||||
- Captures by hour-of-day distribution
|
||||
|
||||
Useful for spotting "qBittorrent appears in 80% of freezes" type patterns.
|
||||
|
||||
## Live graphs
|
||||
|
||||
The GUI's **Live graphs** tab shows the last hour of load, PSI io/cpu, and D-state count as scrolling line graphs. Red dashed lines mark snapshot capture events so you can correlate spikes with the captured forensics.
|
||||
|
||||
## Manual capture
|
||||
|
||||
Click "📸 Capture Now" in the GUI status bar (or in the tray menu) to trigger a snapshot immediately, regardless of whether thresholds are crossed. Useful when you suspect something is brewing but it hasn't tripped the alarms.
|
||||
|
||||
## Future ideas
|
||||
|
||||
- Web UI option (no Qt requirement)
|
||||
- Optional perf-profile capture during event
|
||||
- HTML/JSON snapshot output mode
|
||||
- Snapshot annotations (user notes per capture)
|
||||
- Anonymized export for sharing
|
||||
|
||||
## License
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
# Maintainer: luna <anassaeneroi@pm.me>
|
||||
pkgname=freeze-watcher
|
||||
pkgver=0.2.0
|
||||
pkgver=0.3.0
|
||||
pkgrel=1
|
||||
pkgdesc="Continuously samples system state and snapshots detailed diagnostics when freeze conditions are detected"
|
||||
arch=('any')
|
||||
|
|
|
|||
|
|
@ -1,25 +1,37 @@
|
|||
#!/usr/bin/env python3
|
||||
"""GUI for freeze-watcher service. Shows live metrics and captured snapshots."""
|
||||
"""GUI for freeze-watcher service.
|
||||
|
||||
Layout:
|
||||
- Status bar (service state + Capture Now)
|
||||
- Live metrics row (always visible)
|
||||
- Tabs: Live graphs | Snapshots (filterable list + detail) | Summary
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from collections import Counter, deque
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
from PySide6.QtCore import Qt, QTimer
|
||||
from PySide6.QtGui import QFont, QColor, QAction, QIcon
|
||||
from PySide6.QtCore import Qt, QTimer, QPointF, Signal
|
||||
from PySide6.QtGui import QFont, QColor, QAction, QPainter, QPen, QBrush, QPolygonF
|
||||
from PySide6.QtWidgets import (
|
||||
QApplication, QMainWindow, QWidget, QVBoxLayout, QHBoxLayout,
|
||||
QLabel, QListWidget, QListWidgetItem, QTextEdit, QPushButton,
|
||||
QSplitter, QGroupBox, QGridLayout, QSystemTrayIcon, QMenu, QStyle
|
||||
QSplitter, QGroupBox, QGridLayout, QSystemTrayIcon, QMenu, QStyle,
|
||||
QTabWidget, QLineEdit, QSizePolicy, QMessageBox,
|
||||
)
|
||||
|
||||
LOG_DIR = Path("/var/log/freeze-watcher")
|
||||
TRIGGER = LOG_DIR / ".trigger"
|
||||
SERVICE = "freeze-watcher.service"
|
||||
HISTORY_SECONDS = 3600 # 1h of 1-second samples for graph
|
||||
GRAPH_UPDATE_MS = 1000
|
||||
|
||||
|
||||
# --------------------------------------------------------------------- helpers
|
||||
|
||||
def read_psi(name):
|
||||
try:
|
||||
line = (Path("/proc/pressure") / name).read_text().splitlines()[0]
|
||||
|
|
@ -62,6 +74,8 @@ def service_active():
|
|||
return r.stdout.strip() == "active"
|
||||
|
||||
|
||||
# ----------------------------------------------------------------- widgets
|
||||
|
||||
class MetricLabel(QLabel):
|
||||
def __init__(self, name):
|
||||
super().__init__("--")
|
||||
|
|
@ -76,7 +90,8 @@ class MetricLabel(QLabel):
|
|||
|
||||
def _set_color(self, hex_):
|
||||
self.setStyleSheet(
|
||||
f"background:#1e1e1e; color:{hex_}; border:1px solid #333; border-radius:6px; padding:6px;"
|
||||
f"background:#1e1e1e; color:{hex_}; border:1px solid #333; "
|
||||
f"border-radius:6px; padding:6px;"
|
||||
)
|
||||
|
||||
def set_value(self, value, warn, crit, suffix=""):
|
||||
|
|
@ -89,22 +104,168 @@ class MetricLabel(QLabel):
|
|||
self._set_color("#5cdb95")
|
||||
|
||||
|
||||
class LineGraph(QWidget):
|
||||
"""Simple time-series line graph drawn with QPainter. No extra deps."""
|
||||
|
||||
def __init__(self, label, color="#5cdb95", warn=None, crit=None):
|
||||
super().__init__()
|
||||
self.label = label
|
||||
self.color = QColor(color)
|
||||
self.warn = warn
|
||||
self.crit = crit
|
||||
self.values = deque(maxlen=HISTORY_SECONDS)
|
||||
self.markers = set() # indices to mark as snapshot events
|
||||
self.setMinimumHeight(110)
|
||||
self.setSizePolicy(QSizePolicy.Expanding, QSizePolicy.Preferred)
|
||||
|
||||
def push(self, value, marker=False):
|
||||
self.values.append(value)
|
||||
if marker:
|
||||
self.markers.add(len(self.values) - 1)
|
||||
# Decay markers as values shift out
|
||||
new_markers = set()
|
||||
for m in self.markers:
|
||||
shifted = m - 1 if len(self.values) == HISTORY_SECONDS else m
|
||||
if shifted >= 0:
|
||||
new_markers.add(shifted)
|
||||
self.markers = new_markers
|
||||
self.update()
|
||||
|
||||
def paintEvent(self, _):
|
||||
p = QPainter(self)
|
||||
p.setRenderHint(QPainter.Antialiasing)
|
||||
w, h = self.width(), self.height()
|
||||
|
||||
# Background
|
||||
p.fillRect(0, 0, w, h, QColor("#1a1a1a"))
|
||||
|
||||
if not self.values:
|
||||
p.setPen(QColor("#666"))
|
||||
p.drawText(self.rect(), Qt.AlignCenter, f"{self.label} — no data yet")
|
||||
return
|
||||
|
||||
max_v = max(max(self.values), self.crit or 0, 1)
|
||||
min_v = 0
|
||||
rng = max_v - min_v
|
||||
|
||||
# Title and current value
|
||||
p.setPen(QColor("#aaa"))
|
||||
p.setFont(QFont("monospace", 9))
|
||||
current = self.values[-1]
|
||||
p.drawText(8, 16, f"{self.label} current={current:.1f} max={max(self.values):.1f}")
|
||||
|
||||
# Threshold lines
|
||||
plot_top = 24
|
||||
plot_bot = h - 8
|
||||
plot_h = plot_bot - plot_top
|
||||
for thresh, col in [(self.warn, "#665022"), (self.crit, "#5a2222")]:
|
||||
if thresh is None or thresh > max_v:
|
||||
continue
|
||||
y = plot_bot - (thresh - min_v) / rng * plot_h
|
||||
p.setPen(QPen(QColor(col), 1, Qt.DashLine))
|
||||
p.drawLine(0, int(y), w, int(y))
|
||||
|
||||
# Line
|
||||
n = len(self.values)
|
||||
if n < 2:
|
||||
return
|
||||
step = w / (n - 1) if n > 1 else 0
|
||||
poly = QPolygonF()
|
||||
for i, v in enumerate(self.values):
|
||||
x = i * step
|
||||
y = plot_bot - (v - min_v) / rng * plot_h
|
||||
poly.append(QPointF(x, y))
|
||||
p.setPen(QPen(self.color, 1.5))
|
||||
p.drawPolyline(poly)
|
||||
|
||||
# Markers (snapshot captures)
|
||||
p.setPen(QPen(QColor("#ff5555"), 1, Qt.DotLine))
|
||||
for m in self.markers:
|
||||
x = m * step
|
||||
p.drawLine(int(x), plot_top, int(x), plot_bot)
|
||||
|
||||
|
||||
# ----------------------------------------------------------------- snapshot parsing
|
||||
|
||||
def parse_snapshot(path):
|
||||
"""Extract reason, diagnosis, dstate procs, wchans, top-cpu procs from a snapshot."""
|
||||
try:
|
||||
text = path.read_text(errors="replace")
|
||||
except Exception:
|
||||
return None
|
||||
info = {
|
||||
"path": str(path),
|
||||
"mtime": path.stat().st_mtime,
|
||||
"reason": "",
|
||||
"diagnoses": [],
|
||||
"dstate_procs": [],
|
||||
"wchans": [],
|
||||
"top_cpu_proc": "",
|
||||
}
|
||||
lines = text.splitlines()
|
||||
i = 0
|
||||
while i < len(lines):
|
||||
ln = lines[i]
|
||||
if ln.startswith("=== FREEZE EVENT TRIGGERED:"):
|
||||
info["reason"] = ln.split("TRIGGERED:", 1)[1].strip().rstrip("=").strip()
|
||||
elif ln.startswith("## Diagnosis"):
|
||||
i += 1
|
||||
while i < len(lines) and lines[i].startswith(" •"):
|
||||
info["diagnoses"].append(lines[i].lstrip(" •").strip())
|
||||
i += 1
|
||||
continue
|
||||
elif ln.startswith("## D-state processes"):
|
||||
i += 1
|
||||
while i < len(lines) and not lines[i].startswith("##"):
|
||||
parts = lines[i].split()
|
||||
if len(parts) >= 6 and parts[0] == "D":
|
||||
info["dstate_procs"].append(parts[4]) # comm
|
||||
info["wchans"].append(parts[3])
|
||||
i += 1
|
||||
continue
|
||||
elif ln.startswith("## Top 15 by CPU"):
|
||||
j = i + 1
|
||||
while j < len(lines) and not lines[j].startswith("##"):
|
||||
parts = lines[j].split()
|
||||
if len(parts) >= 6 and parts[1] != "user":
|
||||
info["top_cpu_proc"] = parts[5]
|
||||
break
|
||||
j += 1
|
||||
i += 1
|
||||
return info
|
||||
|
||||
|
||||
# ----------------------------------------------------------------- main window
|
||||
|
||||
class FreezeMonitor(QMainWindow):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.setWindowTitle("Freeze Watcher")
|
||||
self.resize(1100, 700)
|
||||
self.resize(1200, 800)
|
||||
|
||||
central = QWidget()
|
||||
self.setCentralWidget(central)
|
||||
root = QVBoxLayout(central)
|
||||
|
||||
# Status bar
|
||||
# Status row
|
||||
status_row = QHBoxLayout()
|
||||
self.status_label = QLabel()
|
||||
self.status_label.setStyleSheet("padding:8px; font-weight:bold;")
|
||||
root.addWidget(self.status_label)
|
||||
self.status_label.setStyleSheet("padding:6px; font-weight:bold;")
|
||||
status_row.addWidget(self.status_label)
|
||||
status_row.addStretch()
|
||||
self.btn_capture_now = QPushButton("📸 Capture Now")
|
||||
self.btn_capture_now.setToolTip("Manually trigger a snapshot capture")
|
||||
self.btn_capture_now.clicked.connect(self.capture_now)
|
||||
status_row.addWidget(self.btn_capture_now)
|
||||
self.btn_start = QPushButton("Start")
|
||||
self.btn_stop = QPushButton("Stop")
|
||||
self.btn_start.clicked.connect(lambda: self._svc("start"))
|
||||
self.btn_stop.clicked.connect(lambda: self._svc("stop"))
|
||||
status_row.addWidget(self.btn_start)
|
||||
status_row.addWidget(self.btn_stop)
|
||||
root.addLayout(status_row)
|
||||
|
||||
# Live metrics grid
|
||||
# Live metrics row
|
||||
metrics_box = QGroupBox("Live metrics (1s refresh)")
|
||||
grid = QGridLayout(metrics_box)
|
||||
labels = ["PSI IO %", "PSI CPU %", "PSI Mem %", "D-state", "Blocked", "Load (1m)"]
|
||||
|
|
@ -116,24 +277,77 @@ class FreezeMonitor(QMainWindow):
|
|||
grid.addWidget(m, 1, i)
|
||||
root.addWidget(metrics_box)
|
||||
|
||||
# Buttons
|
||||
btn_row = QHBoxLayout()
|
||||
self.btn_start = QPushButton("Start service")
|
||||
self.btn_stop = QPushButton("Stop service")
|
||||
self.btn_refresh = QPushButton("Refresh events")
|
||||
self.btn_open = QPushButton("Open log dir")
|
||||
self.btn_clear = QPushButton("Delete all snapshots")
|
||||
for b in (self.btn_start, self.btn_stop, self.btn_refresh, self.btn_open, self.btn_clear):
|
||||
btn_row.addWidget(b)
|
||||
btn_row.addStretch()
|
||||
self.btn_start.clicked.connect(lambda: self._svc("start"))
|
||||
self.btn_stop.clicked.connect(lambda: self._svc("stop"))
|
||||
self.btn_refresh.clicked.connect(self.load_events)
|
||||
self.btn_open.clicked.connect(lambda: subprocess.Popen(["xdg-open", str(LOG_DIR)]))
|
||||
self.btn_clear.clicked.connect(self.clear_snapshots)
|
||||
root.addLayout(btn_row)
|
||||
# Tabs
|
||||
self.tabs = QTabWidget()
|
||||
root.addWidget(self.tabs, stretch=1)
|
||||
|
||||
self._build_graphs_tab()
|
||||
self._build_snapshots_tab()
|
||||
self._build_summary_tab()
|
||||
|
||||
self._known_events = set()
|
||||
self._all_snapshots = [] # cached parses
|
||||
|
||||
# Tray
|
||||
self.tray = QSystemTrayIcon(self.style().standardIcon(QStyle.SP_ComputerIcon), self)
|
||||
self.tray.setToolTip("Freeze Watcher")
|
||||
menu = QMenu()
|
||||
show_act = QAction("Show window", self)
|
||||
show_act.triggered.connect(self.showNormal)
|
||||
capture_act = QAction("Capture now", self)
|
||||
capture_act.triggered.connect(self.capture_now)
|
||||
quit_act = QAction("Quit", self)
|
||||
quit_act.triggered.connect(QApplication.quit)
|
||||
menu.addAction(show_act)
|
||||
menu.addAction(capture_act)
|
||||
menu.addSeparator()
|
||||
menu.addAction(quit_act)
|
||||
self.tray.setContextMenu(menu)
|
||||
self.tray.activated.connect(self._tray_clicked)
|
||||
self.tray.show()
|
||||
|
||||
# Tick timer
|
||||
self.timer = QTimer()
|
||||
self.timer.timeout.connect(self.tick)
|
||||
self.timer.start(GRAPH_UPDATE_MS)
|
||||
self.load_events()
|
||||
self.tick()
|
||||
|
||||
# ---- tabs
|
||||
|
||||
def _build_graphs_tab(self):
|
||||
tab = QWidget()
|
||||
v = QVBoxLayout(tab)
|
||||
v.addWidget(QLabel("<b>Last hour</b> — red dashed lines mark snapshot captures"))
|
||||
self.graph_load = LineGraph("Load (1m)", "#5cdb95", warn=8, crit=16)
|
||||
self.graph_psi_io = LineGraph("PSI io.some %", "#88ccff", warn=30, crit=60)
|
||||
self.graph_psi_cpu = LineGraph("PSI cpu.some %", "#ffcc66", warn=30, crit=60)
|
||||
self.graph_dstate = LineGraph("D-state procs", "#ff8888", warn=3, crit=6)
|
||||
for g in (self.graph_load, self.graph_psi_io, self.graph_psi_cpu, self.graph_dstate):
|
||||
v.addWidget(g)
|
||||
self.tabs.addTab(tab, "Live graphs")
|
||||
|
||||
def _build_snapshots_tab(self):
|
||||
tab = QWidget()
|
||||
v = QVBoxLayout(tab)
|
||||
|
||||
filter_row = QHBoxLayout()
|
||||
filter_row.addWidget(QLabel("Filter:"))
|
||||
self.filter_edit = QLineEdit()
|
||||
self.filter_edit.setPlaceholderText("substring match against reason/diagnosis (e.g. btrfs, dstate, usb)")
|
||||
self.filter_edit.textChanged.connect(self._apply_filter)
|
||||
filter_row.addWidget(self.filter_edit)
|
||||
self.btn_refresh = QPushButton("Refresh")
|
||||
self.btn_refresh.clicked.connect(self.load_events)
|
||||
self.btn_open = QPushButton("Open log dir")
|
||||
self.btn_open.clicked.connect(lambda: subprocess.Popen(["xdg-open", str(LOG_DIR)]))
|
||||
self.btn_clear = QPushButton("Delete all")
|
||||
self.btn_clear.clicked.connect(self.clear_snapshots)
|
||||
filter_row.addWidget(self.btn_refresh)
|
||||
filter_row.addWidget(self.btn_open)
|
||||
filter_row.addWidget(self.btn_clear)
|
||||
v.addLayout(filter_row)
|
||||
|
||||
# Splitter: list | detail
|
||||
splitter = QSplitter(Qt.Horizontal)
|
||||
self.event_list = QListWidget()
|
||||
self.event_list.itemSelectionChanged.connect(self.show_selected)
|
||||
|
|
@ -142,31 +356,27 @@ class FreezeMonitor(QMainWindow):
|
|||
self.detail.setReadOnly(True)
|
||||
self.detail.setFont(QFont("monospace"))
|
||||
splitter.addWidget(self.detail)
|
||||
splitter.setSizes([350, 750])
|
||||
root.addWidget(splitter, stretch=1)
|
||||
splitter.setSizes([400, 800])
|
||||
v.addWidget(splitter, stretch=1)
|
||||
self.tabs.addTab(tab, "Snapshots")
|
||||
|
||||
self._known_events = set()
|
||||
def _build_summary_tab(self):
|
||||
tab = QWidget()
|
||||
v = QVBoxLayout(tab)
|
||||
top_row = QHBoxLayout()
|
||||
top_row.addWidget(QLabel("<b>Cross-snapshot correlation</b> — patterns across all captures"))
|
||||
top_row.addStretch()
|
||||
btn = QPushButton("Refresh summary")
|
||||
btn.clicked.connect(self.refresh_summary)
|
||||
top_row.addWidget(btn)
|
||||
v.addLayout(top_row)
|
||||
self.summary_text = QTextEdit()
|
||||
self.summary_text.setReadOnly(True)
|
||||
self.summary_text.setFont(QFont("monospace"))
|
||||
v.addWidget(self.summary_text)
|
||||
self.tabs.addTab(tab, "Summary")
|
||||
|
||||
# Tray (created before timer so tick() can use it)
|
||||
self.tray = QSystemTrayIcon(self.style().standardIcon(QStyle.SP_ComputerIcon), self)
|
||||
self.tray.setToolTip("Freeze Watcher")
|
||||
menu = QMenu()
|
||||
show_act = QAction("Show window", self)
|
||||
show_act.triggered.connect(self.showNormal)
|
||||
quit_act = QAction("Quit", self)
|
||||
quit_act.triggered.connect(QApplication.quit)
|
||||
menu.addAction(show_act)
|
||||
menu.addAction(quit_act)
|
||||
self.tray.setContextMenu(menu)
|
||||
self.tray.activated.connect(self._tray_clicked)
|
||||
self.tray.show()
|
||||
|
||||
# Timer
|
||||
self.timer = QTimer()
|
||||
self.timer.timeout.connect(self.tick)
|
||||
self.timer.start(1000)
|
||||
self.load_events()
|
||||
self.tick()
|
||||
# ---- actions
|
||||
|
||||
def _tray_clicked(self, reason):
|
||||
if reason == QSystemTrayIcon.Trigger:
|
||||
|
|
@ -184,6 +394,22 @@ class FreezeMonitor(QMainWindow):
|
|||
subprocess.run(["pkexec", "systemctl", action, SERVICE])
|
||||
self.tick()
|
||||
|
||||
def capture_now(self):
|
||||
try:
|
||||
TRIGGER.write_text("manual (gui)")
|
||||
self.tray.showMessage(
|
||||
"Freeze Watcher",
|
||||
"Manual capture requested",
|
||||
QSystemTrayIcon.Information,
|
||||
2000,
|
||||
)
|
||||
except Exception as e:
|
||||
QMessageBox.warning(self, "Capture failed",
|
||||
f"Could not write trigger file at {TRIGGER}:\n{e}\n\n"
|
||||
"Make sure you're in the group owning /var/log/freeze-watcher.")
|
||||
|
||||
# ---- tick
|
||||
|
||||
def tick(self):
|
||||
psi_io = read_psi("io")
|
||||
psi_cpu = read_psi("cpu")
|
||||
|
|
@ -202,21 +428,28 @@ class FreezeMonitor(QMainWindow):
|
|||
active = service_active()
|
||||
if active:
|
||||
self.status_label.setText("● Service: ACTIVE")
|
||||
self.status_label.setStyleSheet("padding:8px; font-weight:bold; color:#5cdb95;")
|
||||
self.status_label.setStyleSheet("padding:6px; font-weight:bold; color:#5cdb95;")
|
||||
else:
|
||||
self.status_label.setText("● Service: STOPPED")
|
||||
self.status_label.setStyleSheet("padding:8px; font-weight:bold; color:#ff5555;")
|
||||
self.status_label.setStyleSheet("padding:6px; font-weight:bold; color:#ff5555;")
|
||||
|
||||
self.tray.setToolTip(
|
||||
f"Freeze Watcher — {'active' if active else 'stopped'}\n"
|
||||
f"PSI io {int(psi_io)}% / cpu {int(psi_cpu)}% — D:{dstate} blk:{blocked} load:{load:.2f}"
|
||||
)
|
||||
|
||||
# Auto-refresh events list when a new file appears
|
||||
# Detect new snapshot files for graph markers
|
||||
current = set(p.name for p in LOG_DIR.glob("snap-*.log")) if LOG_DIR.exists() else set()
|
||||
if current != self._known_events:
|
||||
new = current - self._known_events
|
||||
is_marker = bool(new)
|
||||
self.graph_load.push(load, marker=is_marker)
|
||||
self.graph_psi_io.push(psi_io, marker=is_marker)
|
||||
self.graph_psi_cpu.push(psi_cpu, marker=is_marker)
|
||||
self.graph_dstate.push(dstate, marker=is_marker)
|
||||
|
||||
if current != self._known_events:
|
||||
self._known_events = current
|
||||
self._all_snapshots = [] # invalidate cache
|
||||
self.load_events()
|
||||
for n in new:
|
||||
self.tray.showMessage(
|
||||
|
|
@ -226,24 +459,34 @@ class FreezeMonitor(QMainWindow):
|
|||
5000,
|
||||
)
|
||||
|
||||
# ---- snapshots tab
|
||||
|
||||
def load_events(self):
|
||||
self.event_list.clear()
|
||||
if not LOG_DIR.exists():
|
||||
return
|
||||
files = sorted(LOG_DIR.glob("snap-*.log"), key=lambda p: p.stat().st_mtime, reverse=True)
|
||||
for p in files:
|
||||
reason = ""
|
||||
try:
|
||||
first = p.read_text().splitlines()[0]
|
||||
if "TRIGGERED:" in first:
|
||||
reason = first.split("TRIGGERED:", 1)[1].strip().rstrip("=").strip()
|
||||
except Exception:
|
||||
pass
|
||||
ts = datetime.fromtimestamp(p.stat().st_mtime).strftime("%Y-%m-%d %H:%M:%S")
|
||||
item = QListWidgetItem(f"{ts}\n {reason}")
|
||||
item.setData(Qt.UserRole, str(p))
|
||||
self.event_list.addItem(item)
|
||||
if not self._all_snapshots:
|
||||
self._all_snapshots = [parse_snapshot(p) for p in files]
|
||||
self._all_snapshots = [s for s in self._all_snapshots if s]
|
||||
self._known_events = set(p.name for p in files)
|
||||
self._apply_filter()
|
||||
|
||||
def _apply_filter(self):
|
||||
needle = self.filter_edit.text().lower().strip()
|
||||
self.event_list.clear()
|
||||
for info in self._all_snapshots:
|
||||
text_blob = (info["reason"] + " " + " ".join(info["diagnoses"])).lower()
|
||||
if needle and needle not in text_blob:
|
||||
continue
|
||||
ts = datetime.fromtimestamp(info["mtime"]).strftime("%Y-%m-%d %H:%M:%S")
|
||||
diag_short = info["diagnoses"][0][:80] if info["diagnoses"] else ""
|
||||
label = f"{ts}\n reason: {info['reason']}"
|
||||
if diag_short:
|
||||
label += f"\n → {diag_short}"
|
||||
item = QListWidgetItem(label)
|
||||
item.setData(Qt.UserRole, info["path"])
|
||||
self.event_list.addItem(item)
|
||||
|
||||
def show_selected(self):
|
||||
items = self.event_list.selectedItems()
|
||||
|
|
@ -258,17 +501,89 @@ class FreezeMonitor(QMainWindow):
|
|||
def clear_snapshots(self):
|
||||
if not LOG_DIR.exists():
|
||||
return
|
||||
from PySide6.QtWidgets import QMessageBox
|
||||
if QMessageBox.question(self, "Delete", "Delete all captured snapshots?") != QMessageBox.Yes:
|
||||
return
|
||||
for p in LOG_DIR.glob("snap-*.log"):
|
||||
try:
|
||||
p.unlink()
|
||||
except Exception:
|
||||
try:
|
||||
subprocess.run(["pkexec", "rm", str(p)])
|
||||
except Exception:
|
||||
pass
|
||||
self._all_snapshots = []
|
||||
self.load_events()
|
||||
self.detail.clear()
|
||||
|
||||
# ---- summary tab
|
||||
|
||||
def refresh_summary(self):
|
||||
if not self._all_snapshots:
|
||||
self.load_events()
|
||||
snaps = self._all_snapshots
|
||||
if not snaps:
|
||||
self.summary_text.setPlainText("No snapshots captured yet.")
|
||||
return
|
||||
|
||||
wchans = Counter()
|
||||
procs = Counter()
|
||||
top_cpu = Counter()
|
||||
diagnoses = Counter()
|
||||
hours = Counter()
|
||||
first = None
|
||||
last = None
|
||||
for s in snaps:
|
||||
for w in s["wchans"]:
|
||||
wchans[w] += 1
|
||||
for p in s["dstate_procs"]:
|
||||
procs[p] += 1
|
||||
if s["top_cpu_proc"]:
|
||||
top_cpu[s["top_cpu_proc"]] += 1
|
||||
for d in s["diagnoses"]:
|
||||
# First clause only, before " — "
|
||||
key = d.split("—")[0].strip()[:60]
|
||||
diagnoses[key] += 1
|
||||
ts = datetime.fromtimestamp(s["mtime"])
|
||||
hours[ts.hour] += 1
|
||||
first = min(first, ts) if first else ts
|
||||
last = max(last, ts) if last else ts
|
||||
|
||||
def topn(ctr, n=10):
|
||||
total = sum(ctr.values()) or 1
|
||||
lines = []
|
||||
for k, v in ctr.most_common(n):
|
||||
pct = 100 * v / total
|
||||
bar = "█" * int(pct / 2)
|
||||
lines.append(f" {v:5d} ({pct:5.1f}%) {bar:<25} {k}")
|
||||
return "\n".join(lines) if lines else " (none)"
|
||||
|
||||
hour_dist = []
|
||||
for h in range(24):
|
||||
count = hours.get(h, 0)
|
||||
bar = "▇" * count
|
||||
hour_dist.append(f" {h:02d}:00 {bar} {count}")
|
||||
|
||||
out = []
|
||||
out.append(f"Snapshots analyzed: {len(snaps)}")
|
||||
if first and last:
|
||||
out.append(f"Time range: {first} → {last}")
|
||||
out.append("")
|
||||
out.append("─── Top diagnoses ───────────────────────────────────────")
|
||||
out.append(topn(diagnoses))
|
||||
out.append("")
|
||||
out.append("─── Top D-state wchans (where stuck) ────────────────────")
|
||||
out.append(topn(wchans))
|
||||
out.append("")
|
||||
out.append("─── Top processes seen in D-state ───────────────────────")
|
||||
out.append(topn(procs))
|
||||
out.append("")
|
||||
out.append("─── Top CPU consumers at capture time ───────────────────")
|
||||
out.append(topn(top_cpu))
|
||||
out.append("")
|
||||
out.append("─── Captures by hour of day ─────────────────────────────")
|
||||
out.append("\n".join(hour_dist))
|
||||
self.summary_text.setPlainText("\n".join(out))
|
||||
|
||||
|
||||
def main():
|
||||
app = QApplication(sys.argv)
|
||||
|
|
|
|||
|
|
@ -1,11 +1,12 @@
|
|||
#!/bin/bash
|
||||
# Continuously samples system state. Writes a detailed snapshot when
|
||||
# anomaly thresholds are crossed (D-states, PSI pressure, blocked procs).
|
||||
# anomaly thresholds are crossed, or when the GUI requests a manual capture.
|
||||
|
||||
set -u
|
||||
|
||||
LOG_DIR=/var/log/freeze-watcher
|
||||
CONFIG=/etc/freeze-watcher.conf
|
||||
TRIGGER_FILE="$LOG_DIR/.trigger"
|
||||
|
||||
# --- defaults (overridden by /etc/freeze-watcher.conf) -------------------
|
||||
PSI_IO_THRESHOLD=40
|
||||
|
|
@ -30,8 +31,6 @@ mkdir -p "$LOG_DIR"
|
|||
last_capture=0
|
||||
captures_since_prune=0
|
||||
|
||||
# Send a desktop notification to each configured user.
|
||||
# Daemon runs as root; we sudo into the user's session for DBus access.
|
||||
notify_users() {
|
||||
[ -z "$NOTIFY_USERS" ] && return 0
|
||||
local body="$1"
|
||||
|
|
@ -48,7 +47,6 @@ notify_users() {
|
|||
done
|
||||
}
|
||||
|
||||
# Delete old snapshots based on MAX_SNAPSHOTS and MAX_SNAPSHOT_AGE_DAYS.
|
||||
prune_snapshots() {
|
||||
if [ "${MAX_SNAPSHOT_AGE_DAYS:-0}" -gt 0 ]; then
|
||||
find "$LOG_DIR" -maxdepth 1 -name 'snap-*.log' -type f \
|
||||
|
|
@ -110,13 +108,108 @@ capture_bpftrace() {
|
|||
' 2>/dev/null | tail -40
|
||||
}
|
||||
|
||||
# Reads a snapshot file and emits one or more diagnosis lines.
|
||||
# Pattern-matches D-state wchans and PSI metrics against known freeze types.
|
||||
classify_snapshot() {
|
||||
local file="$1"
|
||||
local diagnoses=()
|
||||
|
||||
local dstate_section
|
||||
dstate_section=$(awk '/^## D-state processes/{flag=1; next} /^## /{flag=0} flag' "$file")
|
||||
local softirq
|
||||
softirq=$(awk '/^## softirq/{flag=1; next} /^## /{flag=0} flag' "$file")
|
||||
|
||||
local psi_io_some psi_io_full psi_cpu_some psi_mem_some
|
||||
psi_io_some=$(awk '/^\[io\]/{f=1; next} f && /^some/{match($0,/avg10=[0-9.]+/); v=substr($0,RSTART+6,RLENGTH-6); split(v,a,"."); print a[1]+0; exit}' "$file")
|
||||
psi_io_full=$(awk '/^\[io\]/{f=1; next} f && /^full/{match($0,/avg10=[0-9.]+/); v=substr($0,RSTART+6,RLENGTH-6); split(v,a,"."); print a[1]+0; exit}' "$file")
|
||||
psi_cpu_some=$(awk '/^\[cpu\]/{f=1; next} f && /^some/{match($0,/avg10=[0-9.]+/); v=substr($0,RSTART+6,RLENGTH-6); split(v,a,"."); print a[1]+0; exit}' "$file")
|
||||
psi_mem_some=$(awk '/^\[memory\]/{f=1; next} f && /^some/{match($0,/avg10=[0-9.]+/); v=substr($0,RSTART+6,RLENGTH-6); split(v,a,"."); print a[1]+0; exit}' "$file")
|
||||
|
||||
local dcount
|
||||
dcount=$(echo "$dstate_section" | grep -c "^D ")
|
||||
|
||||
# btrfs lock contention
|
||||
if echo "$dstate_section" | grep -qE "btrfs_lock_root_node|btrfs_sync_log"; then
|
||||
if [ "${psi_io_some:-0}" -lt 10 ]; then
|
||||
diagnoses+=("btrfs metadata lock contention — wchans show btrfs_lock_root_node/sync_log with low PSI io. Typical of qgroups + many snapshots, or heavy fsync churn.")
|
||||
else
|
||||
diagnoses+=("btrfs lock contention with concurrent disk I/O")
|
||||
fi
|
||||
fi
|
||||
|
||||
# btrfs metadata pressure (less severe)
|
||||
if echo "$dstate_section" | grep -q "btrfs_search_slot" && \
|
||||
! echo "$dstate_section" | grep -q "btrfs_lock_root_node"; then
|
||||
diagnoses+=("btrfs metadata pressure — cleaner walking trees, fsyncs queueing")
|
||||
fi
|
||||
|
||||
# USB stall
|
||||
if echo "$dstate_section" | grep -qE "usb_sg_wait|scsi_eh|sd_check_events|usb_hcd"; then
|
||||
diagnoses+=("USB storage stall — one external drive responding slowly")
|
||||
fi
|
||||
|
||||
# Memory swap thrash
|
||||
if echo "$dstate_section" | grep -q "folio_wait_bit_common"; then
|
||||
if [ "${psi_mem_some:-0}" -gt 5 ]; then
|
||||
diagnoses+=("memory pressure / swap thrash — folio_wait + PSI mem ${psi_mem_some}%")
|
||||
else
|
||||
diagnoses+=("page-fault stall on swapped-out pages — apps faulting back from zram/swap (consider lower swappiness)")
|
||||
fi
|
||||
fi
|
||||
|
||||
# CPU oversubscription
|
||||
if [ "${psi_cpu_some:-0}" -gt 30 ] && [ "$dcount" -lt 3 ]; then
|
||||
diagnoses+=("CPU oversubscription — PSI cpu ${psi_cpu_some}%, few D-state procs. Heavy compile, transcoding, or runaway process competing for cores.")
|
||||
fi
|
||||
|
||||
# NIC IRQ saturation — use delta against the running baseline so old
|
||||
# cumulative skew doesn't trigger forever.
|
||||
if [ -r "$LOG_DIR/.softirq-baseline" ] && [ -n "$(echo "$softirq" | grep '^ *NET_RX:')" ]; then
|
||||
local cur prev
|
||||
cur=$(echo "$softirq" | grep '^ *NET_RX:' | sed 's/^ *NET_RX://')
|
||||
prev=$(awk '/^ *NET_RX:/{sub(/^ *NET_RX:/, ""); print; exit}' "$LOG_DIR/.softirq-baseline")
|
||||
if [ -n "$prev" ]; then
|
||||
local max=0 total=0 i=1
|
||||
for v in $cur; do
|
||||
[[ "$v" =~ ^[0-9]+$ ]] || { i=$((i+1)); continue; }
|
||||
local pv
|
||||
pv=$(echo "$prev" | awk -v n=$i '{print $n}')
|
||||
[[ "$pv" =~ ^[0-9]+$ ]] || pv=0
|
||||
local d=$((v - pv))
|
||||
[ "$d" -lt 0 ] && d=0
|
||||
[ "$d" -gt "$max" ] && max=$d
|
||||
total=$((total + d))
|
||||
i=$((i+1))
|
||||
done
|
||||
if [ "$total" -gt 100000 ]; then
|
||||
local pct=$((max * 100 / total))
|
||||
if [ "$pct" -gt 70 ]; then
|
||||
diagnoses+=("NIC softirq concentrated on one CPU (${pct}% of recent NET_RX) — enable RPS, install irqbalance")
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# Genuine I/O bottleneck
|
||||
if [ "${psi_io_full:-0}" -gt 30 ]; then
|
||||
diagnoses+=("genuine I/O bottleneck — PSI io full ${psi_io_full}%, device truly can't keep up")
|
||||
fi
|
||||
|
||||
if [ ${#diagnoses[@]} -eq 0 ]; then
|
||||
echo " • uncertain — no known signature matched; review wchans and metrics manually"
|
||||
else
|
||||
for d in "${diagnoses[@]}"; do
|
||||
echo " • $d"
|
||||
done
|
||||
fi
|
||||
}
|
||||
|
||||
capture_snapshot() {
|
||||
local reason="$1"
|
||||
local file="$LOG_DIR/snap-$(date +%Y%m%d-%H%M%S).log"
|
||||
local body="$LOG_DIR/.snap-body-$$.tmp"
|
||||
local final="$LOG_DIR/snap-$(date +%Y%m%d-%H%M%S).log"
|
||||
|
||||
{
|
||||
echo "=== FREEZE EVENT TRIGGERED: $reason ==="
|
||||
echo "Time: $(date -Iseconds)"
|
||||
echo
|
||||
echo "## /proc/loadavg"
|
||||
cat /proc/loadavg
|
||||
echo
|
||||
|
|
@ -166,17 +259,47 @@ capture_snapshot() {
|
|||
echo
|
||||
echo "## journal warnings/errors last 60s"
|
||||
journalctl --since "60 seconds ago" -p warning --no-pager 2>&1 | tail -40
|
||||
} > "$file" 2>&1
|
||||
echo "$(date -Iseconds) Captured snapshot to $file (reason: $reason)" >> "$LOG_DIR/events.log"
|
||||
} > "$body" 2>&1
|
||||
|
||||
# Build final file: header + diagnosis (computed from body) + body
|
||||
{
|
||||
echo "=== FREEZE EVENT TRIGGERED: $reason ==="
|
||||
echo "Time: $(date -Iseconds)"
|
||||
echo
|
||||
echo "## Diagnosis (auto-classified)"
|
||||
classify_snapshot "$body"
|
||||
echo
|
||||
cat "$body"
|
||||
} > "$final"
|
||||
|
||||
rm -f "$body"
|
||||
echo "$(date -Iseconds) Captured snapshot to $final (reason: $reason)" >> "$LOG_DIR/events.log"
|
||||
notify_users "$reason"
|
||||
}
|
||||
|
||||
echo "$(date -Iseconds) freeze-watcher started (pid $$)" >> "$LOG_DIR/events.log"
|
||||
prune_snapshots
|
||||
|
||||
# Initial softirq baseline so classifier can compute deltas
|
||||
cp /proc/softirqs "$LOG_DIR/.softirq-baseline" 2>/dev/null
|
||||
softirq_baseline_at=$(date +%s)
|
||||
|
||||
while true; do
|
||||
now=$(date +%s)
|
||||
|
||||
# Manual trigger from GUI (file-based, no auth required for users in log group)
|
||||
if [ -f "$TRIGGER_FILE" ]; then
|
||||
manual_reason="manual"
|
||||
if [ -s "$TRIGGER_FILE" ]; then
|
||||
manual_reason=$(head -c 100 "$TRIGGER_FILE" | tr -d '\n')
|
||||
fi
|
||||
rm -f "$TRIGGER_FILE"
|
||||
capture_snapshot "$manual_reason" &
|
||||
last_capture=$now
|
||||
sleep "$SAMPLE_INTERVAL"
|
||||
continue
|
||||
fi
|
||||
|
||||
psi_io=$(awk '/^some/{for(i=1;i<=NF;i++) if($i~/^avg10=/){gsub(/avg10=/,"",$i); print int($i+0.5); exit}}' /proc/pressure/io 2>/dev/null)
|
||||
psi_cpu=$(awk '/^some/{for(i=1;i<=NF;i++) if($i~/^avg10=/){gsub(/avg10=/,"",$i); print int($i+0.5); exit}}' /proc/pressure/cpu 2>/dev/null)
|
||||
dstate=$(ps -eo state --no-headers | grep -c "^D")
|
||||
|
|
@ -200,5 +323,11 @@ while true; do
|
|||
fi
|
||||
fi
|
||||
|
||||
# Refresh softirq baseline every 5 minutes so classifier deltas reflect recent activity
|
||||
if [ $((now - softirq_baseline_at)) -ge 300 ]; then
|
||||
cp /proc/softirqs "$LOG_DIR/.softirq-baseline" 2>/dev/null
|
||||
softirq_baseline_at=$now
|
||||
fi
|
||||
|
||||
sleep "$SAMPLE_INTERVAL"
|
||||
done
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue