feat(go): port FIM/pkgdb/rootcheck integrity engines and add go-sidecar dashboard consumer
Adds sidecar-only, --state-dir-gated FIM, package-DB-anchor, and rootcheck
engines to the Go migration sidecar, wired into agent.Sweep/Initialize
behind the existing baseline lifecycle. Adds a read-only, schema-checked
Python dashboard consumer (go_sidecar_state_dir, /api/go-sidecar/*, Sidecar
tab) that never starts, stops, or mutates the Go sidecar's state.
Also fixes drift found while reconciling this work: enodia_sentinel/web.py
had reinvented local schema constants instead of using the canonical
enodia_sentinel/schemas.py catalog (now registers enodia.go.sidecar.v1
there and reuses ALERT_SNAPSHOT_V1/INCIDENT_V1); docs/RULES.md had drifted
from what `rules docs` actually generates for SIDs 100069-100078 (ruleops.py
was missing metadata for four SIDs and had stale drill text for four more),
now back in sync with a regression test pinning them together.
Updates CLAUDE.md, README.md, go-agent/README.md, and docs/{ROADMAP,
GO_PORT_HANDOFF,SURICATA_ASSIMILATION,THREAT_MODEL,OPERATIONS,SCHEMAS,
COMMAND_REFERENCE}.md to reflect the landed work.
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01NQivSKBqQJsayz1xcYqWzZ
This commit is contained in:
Luna
parent
1d1dee7f6e
commit
d835386381
43 changed files with 3419 additions and 72 deletions
|
|
@ -175,6 +175,199 @@ def tail_events(cfg: Config, limit: int = 100) -> list[str]:
|
|||
return lines[-limit:]
|
||||
|
||||
|
||||
# --- isolated Go sidecar evidence (read-only migration consumer) ----------
|
||||
|
||||
def _go_sidecar_dir(cfg: Config) -> Path | None:
|
||||
# This is an opt-in evidence boundary, not an alternate runtime root. The
|
||||
# web process only derives paths beneath it and never creates or mutates it.
|
||||
raw = cfg.go_sidecar_state_dir.strip()
|
||||
return Path(raw) if raw else None
|
||||
|
||||
|
||||
def go_sidecar_status(cfg: Config, now: float | None = None) -> dict:
|
||||
"""Describe the separately configured Go evidence tree without touching it.
|
||||
|
||||
The Python daemon remains authoritative. This endpoint deliberately exposes
|
||||
only whether the optional migration sidecar is configured and whether its
|
||||
retained heartbeat is fresh; it never starts, stops, or probes the service.
|
||||
"""
|
||||
directory = _go_sidecar_dir(cfg)
|
||||
report = {
|
||||
"schema": schemas.GO_SIDECAR_V1,
|
||||
"read_only": True,
|
||||
"configured": directory is not None,
|
||||
"state_dir": str(directory) if directory else "",
|
||||
"available": bool(directory and directory.is_dir()),
|
||||
"heartbeat": {"status": "not-configured"},
|
||||
}
|
||||
if directory is None:
|
||||
return report
|
||||
heartbeat = directory / "heartbeat"
|
||||
try:
|
||||
timestamp = int(heartbeat.read_text().strip())
|
||||
except (OSError, ValueError):
|
||||
report["heartbeat"] = {"status": "missing"}
|
||||
return report
|
||||
ts = time.time() if now is None else now
|
||||
age = max(0.0, ts - timestamp)
|
||||
report["heartbeat"] = {
|
||||
"status": "fresh" if age <= cfg.heartbeat_max_age else "stale",
|
||||
"timestamp": timestamp,
|
||||
"age_seconds": age,
|
||||
"max_age_seconds": cfg.heartbeat_max_age,
|
||||
}
|
||||
return report
|
||||
|
||||
|
||||
def _go_snapshot_report(path: Path) -> dict | None:
|
||||
# Sidecar files are retained forensic input, not trusted application data.
|
||||
# Require the expected schema before presenting an arbitrary JSON document
|
||||
# as a Sentinel alert snapshot.
|
||||
try:
|
||||
report = json.loads(path.read_text())
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
if not isinstance(report, dict) or report.get("schema") != schemas.ALERT_SNAPSHOT_V1:
|
||||
return None
|
||||
return report
|
||||
|
||||
|
||||
def list_go_sidecar_alerts(cfg: Config, limit: int = 200) -> list[dict]:
|
||||
directory = _go_sidecar_dir(cfg)
|
||||
if directory is None:
|
||||
return []
|
||||
alerts = []
|
||||
for path in sorted(directory.glob("alert-*.json"), reverse=True)[:limit]:
|
||||
report = _go_snapshot_report(path)
|
||||
if report is None:
|
||||
continue
|
||||
records = report.get("alerts", [])
|
||||
if not isinstance(records, list):
|
||||
continue
|
||||
alerts.append({
|
||||
"name": path.with_suffix(".log").name,
|
||||
"time": report.get("time", ""),
|
||||
"host": report.get("host", ""),
|
||||
"severity": report.get("severity", ""),
|
||||
"signatures": sorted({a.get("signature", "") for a in records
|
||||
if isinstance(a, dict)}),
|
||||
"sids": sorted({a.get("sid", 0) for a in records
|
||||
if isinstance(a, dict) and a.get("sid")}),
|
||||
"count": len(records),
|
||||
"source": "go-sidecar",
|
||||
})
|
||||
return alerts
|
||||
|
||||
|
||||
def get_go_sidecar_alert(cfg: Config, name: str) -> dict | None:
|
||||
if not isinstance(name, str) or not _ALERT_NAME.match(name):
|
||||
return None
|
||||
directory = _go_sidecar_dir(cfg)
|
||||
if directory is None:
|
||||
return None
|
||||
base = directory / name
|
||||
report = _go_snapshot_report(base.with_suffix(".json"))
|
||||
if report is None:
|
||||
return None
|
||||
data = {"json": report, "source": "go-sidecar"}
|
||||
try:
|
||||
if base.is_file():
|
||||
data["text"] = base.read_text()
|
||||
except OSError:
|
||||
pass
|
||||
return data
|
||||
|
||||
|
||||
def go_sidecar_events(cfg: Config, limit: int = 100) -> dict:
|
||||
"""Return valid retained envelopes, failing closed if either segment corrupts."""
|
||||
directory = _go_sidecar_dir(cfg)
|
||||
result = {"events": [], "source": "go-sidecar", "error": None}
|
||||
if directory is None:
|
||||
return result
|
||||
# Match the Go reader's chronology: the rotated segment precedes its active
|
||||
# successor. Do not return a partial stream after corruption, because that
|
||||
# could make an incomplete evidence chain look authoritative.
|
||||
rows = []
|
||||
for path in (directory / "events.jsonl.1", directory / "events.jsonl"):
|
||||
try:
|
||||
lines = path.read_text().splitlines()
|
||||
except FileNotFoundError:
|
||||
continue
|
||||
except OSError as exc:
|
||||
result["error"] = f"cannot read retained events: {exc}"
|
||||
return result
|
||||
for line in lines:
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
row = json.loads(line)
|
||||
except ValueError:
|
||||
result["error"] = "retained event log is corrupt"
|
||||
return result
|
||||
if not isinstance(row, dict):
|
||||
result["error"] = "retained event log contains a non-object record"
|
||||
return result
|
||||
rows.append(row)
|
||||
result["events"] = rows[-max(0, limit):] if limit else []
|
||||
return result
|
||||
|
||||
|
||||
def list_go_sidecar_incidents(cfg: Config) -> list[dict]:
|
||||
directory = _go_sidecar_dir(cfg)
|
||||
if directory is None:
|
||||
return []
|
||||
try:
|
||||
index = json.loads((directory / "incidents.json").read_text())
|
||||
except (OSError, ValueError):
|
||||
return []
|
||||
if not isinstance(index, dict):
|
||||
return []
|
||||
# Atomic index replacement protects writers, but retained files can still
|
||||
# be stale or manually damaged; schema filtering keeps this read-only view
|
||||
# from interpreting unrelated JSON as an incident.
|
||||
records = [item for item in index.values()
|
||||
if isinstance(item, dict) and item.get("schema") == schemas.INCIDENT_V1]
|
||||
|
||||
def last_timestamp(item: dict) -> float:
|
||||
try:
|
||||
return float(item.get("last_ts", 0.0))
|
||||
except (TypeError, ValueError):
|
||||
return 0.0
|
||||
|
||||
return sorted(records, key=last_timestamp, reverse=True)
|
||||
|
||||
|
||||
def get_go_sidecar_incident(cfg: Config, incident_id: str) -> dict | None:
|
||||
incident = next((item for item in list_go_sidecar_incidents(cfg)
|
||||
if item.get("id") == incident_id), None)
|
||||
if incident is None:
|
||||
return None
|
||||
snapshots, timeline = [], []
|
||||
for name in incident.get("snapshots", []):
|
||||
if not isinstance(name, str):
|
||||
timeline.append({"snapshot": str(name), "time": "?", "missing": True})
|
||||
continue
|
||||
alert = get_go_sidecar_alert(cfg, name)
|
||||
if alert is None:
|
||||
timeline.append({"snapshot": name, "time": "?", "missing": True})
|
||||
continue
|
||||
report = alert["json"]
|
||||
snapshots.append(report)
|
||||
timeline.append({
|
||||
"snapshot": name,
|
||||
"time": report.get("time", "?"),
|
||||
"severity": report.get("severity", "?"),
|
||||
"signatures": list(dict.fromkeys(
|
||||
row.get("signature", "") for row in report.get("alerts", [])
|
||||
if isinstance(row, dict))),
|
||||
"pids": [row.get("pid") for row in report.get("processes", [])
|
||||
if isinstance(row, dict) and isinstance(row.get("pid"), int)],
|
||||
})
|
||||
timeline.sort(key=lambda row: row.get("time", ""))
|
||||
return {"schema": schemas.INCIDENT_VIEW_V1, "incident": incident,
|
||||
"timeline": timeline, "snapshots": snapshots, "source": "go-sidecar"}
|
||||
|
||||
|
||||
def daemon_status(cfg: Config) -> dict:
|
||||
pid_alive = False
|
||||
pidfile = cfg.log_dir / "sentinel.pid"
|
||||
|
|
@ -505,6 +698,21 @@ class _Handler(BaseHTTPRequestHandler):
|
|||
self._json_or_not_found(alert, send_body=send_body)
|
||||
elif path == "/api/events":
|
||||
self._json({"events": tail_events(cfg)}, send_body=send_body)
|
||||
elif path == "/api/go-sidecar/status":
|
||||
self._json(go_sidecar_status(cfg), send_body=send_body)
|
||||
elif path == "/api/go-sidecar/alerts":
|
||||
self._json(list_go_sidecar_alerts(cfg), send_body=send_body)
|
||||
elif path.startswith("/api/go-sidecar/alerts/"):
|
||||
alert = get_go_sidecar_alert(cfg, path.rsplit("/", 1)[-1])
|
||||
self._json_or_not_found(alert, send_body=send_body)
|
||||
elif path == "/api/go-sidecar/events":
|
||||
self._json(go_sidecar_events(cfg), send_body=send_body)
|
||||
elif path == "/api/go-sidecar/incidents":
|
||||
self._json(list_go_sidecar_incidents(cfg), send_body=send_body)
|
||||
elif path.startswith("/api/go-sidecar/incidents/"):
|
||||
iid = unquote(path.rsplit("/", 1)[-1])
|
||||
report = get_go_sidecar_incident(cfg, iid)
|
||||
self._json_or_not_found(report, send_body=send_body)
|
||||
elif path == "/api/posture":
|
||||
self._json(posture_report(cfg), send_body=send_body)
|
||||
elif path == "/api/rules":
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue