From ea6a0ea6758b93262acd8075406171a6503d5a4d Mon Sep 17 00:00:00 2001 From: luna Date: Sun, 10 May 2026 21:56:05 -0700 Subject: [PATCH] 0.4.0: JSON/HTML snapshot output via freeze-snapshot-convert; GUI 'Open in browser' button --- Makefile | 6 +- README.md | 15 ++ config/freeze-watcher.conf | 7 + packaging/PKGBUILD | 2 +- src/freeze-monitor-gui | 39 +++- src/freeze-snapshot-convert | 391 ++++++++++++++++++++++++++++++++++++ src/freeze-watcher.sh | 11 + 7 files changed, 466 insertions(+), 5 deletions(-) create mode 100755 src/freeze-snapshot-convert diff --git a/Makefile b/Makefile index cdd1fad..5a90ac3 100644 --- a/Makefile +++ b/Makefile @@ -9,8 +9,9 @@ LOG_GROUP ?= $(shell id -gn) .PHONY: install uninstall enable disable status logs clean install: - install -Dm755 src/freeze-watcher.sh $(DESTDIR)$(BINDIR)/freeze-watcher.sh - install -Dm755 src/freeze-monitor-gui $(DESTDIR)$(BINDIR)/freeze-monitor-gui + install -Dm755 src/freeze-watcher.sh $(DESTDIR)$(BINDIR)/freeze-watcher.sh + install -Dm755 src/freeze-monitor-gui $(DESTDIR)$(BINDIR)/freeze-monitor-gui + install -Dm755 src/freeze-snapshot-convert $(DESTDIR)$(BINDIR)/freeze-snapshot-convert install -Dm644 systemd/freeze-watcher.service $(DESTDIR)$(SYSTEMDDIR)/freeze-watcher.service install -Dm644 desktop/freeze-monitor.desktop $(DESTDIR)$(APPDIR)/freeze-monitor.desktop @if [ ! -e "$(DESTDIR)$(CONFDIR)/freeze-watcher.conf" ]; then \ @@ -26,6 +27,7 @@ install: uninstall: rm -f $(DESTDIR)$(BINDIR)/freeze-watcher.sh rm -f $(DESTDIR)$(BINDIR)/freeze-monitor-gui + rm -f $(DESTDIR)$(BINDIR)/freeze-snapshot-convert rm -f $(DESTDIR)$(SYSTEMDDIR)/freeze-watcher.service rm -f $(DESTDIR)$(APPDIR)/freeze-monitor.desktop rm -f $(DESTDIR)$(CONFDIR)/freeze-watcher.conf.new diff --git a/README.md b/README.md index c85e7ed..f11d177 100644 --- a/README.md +++ b/README.md @@ -150,6 +150,21 @@ The GUI's **Live graphs** tab shows the last hour of load, PSI io/cpu, and D-sta Click "📸 Capture Now" in the GUI status bar (or in the tray menu) to trigger a snapshot immediately, regardless of whether thresholds are crossed. Useful when you suspect something is brewing but it hasn't tripped the alarms. +## JSON / HTML output + +Set `OUTPUT_JSON=1` and/or `OUTPUT_HTML=1` in `/etc/freeze-watcher.conf` to write structured sidecars next to each text snapshot: + +- **JSON** (`snap-*.json`) — fully structured: PSI, dstate procs, kernel stacks, memory, top procs, cgroup PSI, softirqs, etc. Easy to consume programmatically. +- **HTML** (`snap-*.html`) — self-contained styled report with color-coded metrics, collapsible sections, and clickable kernel stacks per D-state proc. Useful for sharing or pasting into a bug report. + +You can also convert any existing snapshot manually: + +``` +freeze-snapshot-convert /var/log/freeze-watcher/snap-20260510-214207.log --json --html +``` + +The GUI's "Open in browser" button on the Snapshots tab opens the HTML sidecar of the selected snapshot (generating it on demand if missing). + ## Future ideas - Web UI option (no Qt requirement) diff --git a/config/freeze-watcher.conf b/config/freeze-watcher.conf index 20bd6d2..2c66187 100644 --- a/config/freeze-watcher.conf +++ b/config/freeze-watcher.conf @@ -29,3 +29,10 @@ NOTIFY_URGENCY=normal # low, normal, critical CAPTURE_KERNEL_STACKS=1 # dump /proc/[pid]/stack for D-state procs (free, very useful) CAPTURE_BPFTRACE=0 # run bpftrace one-shot during capture (requires bpftrace) CAPTURE_CGROUP_PSI=1 # walk /sys/fs/cgroup for top PSI cgroups + +# --- output formats ------------------------------------------------------ +# Text snapshot is always written. These add sidecar files (.json / .html) +# next to each snap-*.log. The HTML version has color-coded metrics and +# collapsible sections, useful for sharing or browsing. +OUTPUT_JSON=0 # write snap-*.json +OUTPUT_HTML=0 # write snap-*.html diff --git a/packaging/PKGBUILD b/packaging/PKGBUILD index 206f8b6..6d15351 100644 --- a/packaging/PKGBUILD +++ b/packaging/PKGBUILD @@ -1,6 +1,6 @@ # Maintainer: luna pkgname=freeze-watcher -pkgver=0.3.0 +pkgver=0.4.0 pkgrel=1 pkgdesc="Continuously samples system state and snapshots detailed diagnostics when freeze conditions are detected" arch=('any') diff --git a/src/freeze-monitor-gui b/src/freeze-monitor-gui index 9d33a61..e28c2ff 100755 --- a/src/freeze-monitor-gui +++ b/src/freeze-monitor-gui @@ -339,11 +339,15 @@ class FreezeMonitor(QMainWindow): filter_row.addWidget(self.filter_edit) self.btn_refresh = QPushButton("Refresh") self.btn_refresh.clicked.connect(self.load_events) + self.btn_html = QPushButton("Open in browser") + self.btn_html.setToolTip("Open the HTML sidecar of the selected snapshot (requires OUTPUT_HTML=1)") + self.btn_html.clicked.connect(self.open_html) self.btn_open = QPushButton("Open log dir") self.btn_open.clicked.connect(lambda: subprocess.Popen(["xdg-open", str(LOG_DIR)])) self.btn_clear = QPushButton("Delete all") self.btn_clear.clicked.connect(self.clear_snapshots) filter_row.addWidget(self.btn_refresh) + filter_row.addWidget(self.btn_html) filter_row.addWidget(self.btn_open) filter_row.addWidget(self.btn_clear) v.addLayout(filter_row) @@ -492,12 +496,43 @@ class FreezeMonitor(QMainWindow): items = self.event_list.selectedItems() if not items: return - path = items[0].data(Qt.UserRole) + path = Path(items[0].data(Qt.UserRole)) try: - self.detail.setPlainText(Path(path).read_text()) + text = path.read_text() + html_sidecar = path.with_suffix(".html") + json_sidecar = path.with_suffix(".json") + extras = [] + if html_sidecar.exists(): + extras.append(f"[HTML available: {html_sidecar.name} — right-click 'Open in browser']") + if json_sidecar.exists(): + extras.append(f"[JSON available: {json_sidecar.name}]") + if extras: + text = "\n".join(extras) + "\n" + ("─" * 60) + "\n" + text + self.detail.setPlainText(text) + self._current_snapshot_path = path except Exception as e: self.detail.setPlainText(f"Error: {e}") + def open_html(self): + items = self.event_list.selectedItems() + if not items: + QMessageBox.information(self, "Select a snapshot", + "Pick a snapshot from the list first.") + return + path = Path(items[0].data(Qt.UserRole)) + html = path.with_suffix(".html") + if not html.exists(): + # Try generating on demand + try: + subprocess.run(["freeze-snapshot-convert", "--html", str(path)], + check=True, capture_output=True) + except Exception as e: + QMessageBox.warning(self, "No HTML", + f"HTML sidecar not found and on-demand generation failed:\n{e}\n\n" + "Set OUTPUT_HTML=1 in /etc/freeze-watcher.conf to auto-generate.") + return + subprocess.Popen(["xdg-open", str(html)]) + def clear_snapshots(self): if not LOG_DIR.exists(): return diff --git a/src/freeze-snapshot-convert b/src/freeze-snapshot-convert new file mode 100755 index 0000000..1583cee --- /dev/null +++ b/src/freeze-snapshot-convert @@ -0,0 +1,391 @@ +#!/usr/bin/env python3 +"""Convert a freeze-watcher text snapshot to JSON and/or HTML. + +Usage: + freeze-snapshot-convert PATH.log [--json] [--html] + +Output files are written alongside the input with .json / .html suffixes. +""" + +import argparse +import html as html_lib +import json +import re +import sys +from pathlib import Path + + +# --------------------------------------------------------------------- parser + +def parse_snapshot(text): + """Parse a freeze-watcher text snapshot into a structured dict.""" + data = { + "trigger_reason": "", + "timestamp": "", + "diagnoses": [], + "loadavg": {}, + "procs_running": None, + "procs_blocked": None, + "psi": {}, + "cgroup_psi": {"io": [], "cpu": []}, + "dstate_processes": [], + "kernel_stacks": {}, + "top_cpu": [], + "top_rss": [], + "memory": {}, + "iostat_raw": "", + "interrupts": [], + "softirqs": {}, + "dmesg_recent": [], + "journal_warnings": [], + } + + sections = _split_sections(text) + + # Header + for ln in sections.get("__header__", []): + m = re.match(r"=== FREEZE EVENT TRIGGERED:\s*(.+?)\s*===", ln) + if m: + data["trigger_reason"] = m.group(1) + m = re.match(r"Time:\s*(.+)", ln) + if m: + data["timestamp"] = m.group(1).strip() + + # Diagnosis + for ln in sections.get("Diagnosis (auto-classified)", []): + s = ln.strip() + if s.startswith("•"): + data["diagnoses"].append(s.lstrip("• ").strip()) + + # /proc/loadavg + for ln in sections.get("/proc/loadavg", []): + parts = ln.split() + if len(parts) >= 3: + try: + data["loadavg"] = {"1m": float(parts[0]), "5m": float(parts[1]), "15m": float(parts[2])} + except ValueError: + pass + break + + # /proc/stat + for ln in sections.get("/proc/stat (procs_running, procs_blocked)", []): + m = re.match(r"procs_running\s+(\d+)", ln) + if m: + data["procs_running"] = int(m.group(1)) + m = re.match(r"procs_blocked\s+(\d+)", ln) + if m: + data["procs_blocked"] = int(m.group(1)) + + # PSI + for ln in sections.get("PSI cpu/io/memory", []): + m = re.match(r"\[(\w+)\]", ln) + if m: + current_psi = m.group(1) + data["psi"][current_psi] = {} + continue + m = re.match(r"(some|full)\s+(.+)", ln) + if m and "psi" in data and data["psi"]: + kind = m.group(1) + kvs = dict(re.findall(r"(\w+)=([\d.]+)", m.group(2))) + last_key = list(data["psi"].keys())[-1] + data["psi"][last_key][kind] = {k: float(v) for k, v in kvs.items()} + + # Cgroup PSI + for header, key in [("Top cgroups by PSI io.some avg10", "io"), + ("Top cgroups by PSI cpu.some avg10", "cpu")]: + for ln in sections.get(header, []): + ln = ln.strip() + if not ln: + continue + parts = ln.split(None, 1) + if len(parts) == 2: + try: + data["cgroup_psi"][key].append({"value": float(parts[0]), "cgroup": parts[1]}) + except ValueError: + pass + + # D-state processes + for ln in sections.get("D-state processes (with kernel wchan)", []): + parts = ln.split(None, 5) + if len(parts) >= 5 and parts[0] == "D": + entry = { + "pid": int(parts[1]) if parts[1].isdigit() else parts[1], + "user": parts[2], + "wchan": parts[3], + "comm": parts[4], + "args": parts[5] if len(parts) > 5 else "", + } + data["dstate_processes"].append(entry) + + # Kernel stacks + current_pid = None + for ln in sections.get("Kernel stack traces of D-state procs", []): + m = re.match(r"--- pid=(\d+) comm=(\S+) ---", ln) + if m: + current_pid = int(m.group(1)) + data["kernel_stacks"][current_pid] = {"comm": m.group(2), "stack": []} + elif current_pid and ln.strip().startswith("[<"): + data["kernel_stacks"][current_pid]["stack"].append(ln.strip()) + elif current_pid and ln.strip(): + data["kernel_stacks"][current_pid]["stack"].append(ln.strip()) + + # Top CPU + for ln in sections.get("Top 15 by CPU", []): + parts = ln.split() + if len(parts) >= 6 and parts[1] != "user": + try: + data["top_cpu"].append({ + "pid": int(parts[0]), "user": parts[1], + "pcpu": float(parts[2]), "pmem": float(parts[3]), + "state": parts[4], "comm": parts[5], + }) + except ValueError: + pass + + # Top RSS + for ln in sections.get("Top 15 by RSS memory", []): + parts = ln.split() + if len(parts) >= 6 and parts[1] != "user": + try: + data["top_rss"].append({ + "pid": int(parts[0]), "user": parts[1], + "pcpu": float(parts[2]), "pmem": float(parts[3]), + "rss_kb": int(parts[4]), "comm": parts[5], + }) + except ValueError: + pass + + # /proc/meminfo + for ln in sections.get("/proc/meminfo summary", []): + m = re.match(r"(\w+):\s+(\d+)\s*kB", ln) + if m: + data["memory"][m.group(1)] = int(m.group(2)) + + # iostat raw + data["iostat_raw"] = "\n".join(sections.get("iostat (1s snapshot)", [])) + + # Interrupts (top by total) + for ln in sections.get("interrupts (top by total)", []): + parts = ln.split() + if len(parts) >= 3 and parts[0].isdigit(): + data["interrupts"].append({ + "total": int(parts[0]), + "raw": " ".join(parts[1:]), + }) + + # softirqs - keep raw + softirq_lines = sections.get("softirq", []) + if softirq_lines: + cpu_header = softirq_lines[0].split() if softirq_lines else [] + for ln in softirq_lines[1:]: + m = re.match(r"\s*(\w+):\s+(.+)", ln) + if m: + vals = m.group(2).split() + try: + data["softirqs"][m.group(1)] = [int(v) for v in vals] + except ValueError: + pass + + data["dmesg_recent"] = sections.get("Recent dmesg (last 50 lines)", []) + data["journal_warnings"] = sections.get("journal warnings/errors last 60s", []) + + return data + + +def _split_sections(text): + """Split a snapshot into sections keyed by '## Header' titles.""" + sections = {"__header__": []} + current = "__header__" + for ln in text.splitlines(): + m = re.match(r"## (.+)", ln) + if m: + current = m.group(1).strip() + sections[current] = [] + else: + sections[current].append(ln) + return sections + + +# --------------------------------------------------------------------- HTML + +def render_html(data, source_text): + def metric_class(value, warn, crit): + if value is None: + return "ok" + if value >= crit: + return "crit" + if value >= warn: + return "warn" + return "ok" + + def fmt_psi(kind, key="some"): + block = data["psi"].get(kind, {}).get(key, {}) + return block.get("avg10", 0.0) + + psi_io = fmt_psi("io") + psi_cpu = fmt_psi("cpu") + psi_mem = fmt_psi("memory") + load_1m = data["loadavg"].get("1m", 0) + + diag_html = "".join(f"
  • {html_lib.escape(d)}
    • " for d in data["diagnoses"]) \ + or "
  • uncertain
    • " + + dstate_html = "" + for p in data["dstate_processes"]: + dstate_html += ( + f'
    ' + f'pid {p["pid"]} ({html_lib.escape(p["comm"])}) ' + f'user={html_lib.escape(p["user"])} ' + f'wchan={html_lib.escape(p["wchan"])}' + ) + if p["pid"] in data["kernel_stacks"]: + stack = "\n".join(data["kernel_stacks"][p["pid"]]["stack"]) + dstate_html += f'
    kernel stack
    {html_lib.escape(stack)}
    ' + dstate_html += "
    " + if not dstate_html: + dstate_html = "

    None at capture time.

    " + + top_cpu_rows = "".join( + f"{p['pid']}{html_lib.escape(p['user'])}" + f"{p['pcpu']}{p['pmem']}" + f"{p['state']}{html_lib.escape(p['comm'])}" + for p in data["top_cpu"][:10] + ) + + top_rss_rows = "".join( + f"{p['pid']}{html_lib.escape(p['user'])}" + f"{p['rss_kb']/1024:.0f} MB" + f"{html_lib.escape(p['comm'])}" + for p in data["top_rss"][:10] + ) + + cg_io_rows = "".join( + f"{c['value']}{html_lib.escape(c['cgroup'])}" + for c in data["cgroup_psi"]["io"][:10] + ) or "none" + + cg_cpu_rows = "".join( + f"{c['value']}{html_lib.escape(c['cgroup'])}" + for c in data["cgroup_psi"]["cpu"][:10] + ) or "none" + + return f""" + + +Freeze snapshot — {html_lib.escape(data['timestamp'])} + + +

    Freeze snapshot

    +

    {html_lib.escape(data['timestamp'])} — + trigger: {html_lib.escape(data['trigger_reason'])}

    + +
    +

    Diagnosis

    +
      {diag_html}
    +
    + +
    +

    System pressure

    +
    + {psi_io:.1f}%PSI io.some + {psi_cpu:.1f}%PSI cpu.some + {psi_mem:.1f}%PSI mem.some + {load_1m:.2f}load 1m + {len(data['dstate_processes'])}D-state procs + {data['procs_blocked'] or 0}procs_blocked +
    +
    + +
    +

    D-state processes — where they're stuck

    +{dstate_html} +
    + +
    +

    Top processes by CPU

    + + +{top_cpu_rows} +
    PIDUser%CPU%MEMStateCommand
    +
    + +
    +

    Top processes by RSS memory

    + + +{top_rss_rows} +
    PIDUserRSSCommand
    +
    + +
    +

    Top cgroups by io.pressure

    +{cg_io_rows}
    avg10 %cgroup
    +

    Top cgroups by cpu.pressure

    +{cg_cpu_rows}
    avg10 %cgroup
    +
    + +
    +Raw snapshot text (full contents) +
    {html_lib.escape(source_text)}
    +
    + +""" + + +# --------------------------------------------------------------------- main + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("snapshot", help="Path to snap-*.log file") + ap.add_argument("--json", action="store_true", help="Write JSON sidecar") + ap.add_argument("--html", action="store_true", help="Write HTML sidecar") + args = ap.parse_args() + + if not (args.json or args.html): + ap.error("Specify --json and/or --html") + + src = Path(args.snapshot) + text = src.read_text(errors="replace") + data = parse_snapshot(text) + + if args.json: + out = src.with_suffix(".json") + out.write_text(json.dumps(data, indent=2)) + print(f"wrote {out}", file=sys.stderr) + + if args.html: + out = src.with_suffix(".html") + out.write_text(render_html(data, text)) + print(f"wrote {out}", file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/src/freeze-watcher.sh b/src/freeze-watcher.sh index 2410e3c..dc52a95 100755 --- a/src/freeze-watcher.sh +++ b/src/freeze-watcher.sh @@ -23,6 +23,8 @@ NOTIFY_URGENCY=normal CAPTURE_KERNEL_STACKS=1 CAPTURE_BPFTRACE=0 CAPTURE_CGROUP_PSI=1 +OUTPUT_JSON=0 +OUTPUT_HTML=0 [ -f "$CONFIG" ] && . "$CONFIG" @@ -274,6 +276,15 @@ capture_snapshot() { rm -f "$body" echo "$(date -Iseconds) Captured snapshot to $final (reason: $reason)" >> "$LOG_DIR/events.log" + + # Optional sidecar formats + local convert_args="" + [ "${OUTPUT_JSON:-0}" = "1" ] && convert_args="$convert_args --json" + [ "${OUTPUT_HTML:-0}" = "1" ] && convert_args="$convert_args --html" + if [ -n "$convert_args" ] && command -v freeze-snapshot-convert >/dev/null 2>&1; then + freeze-snapshot-convert $convert_args "$final" 2>/dev/null & + fi + notify_users "$reason" }